testing01.QuickStart.java Source code

Introduction

Here is the source code for testing01.QuickStart.java

Source

/*
 * ====================================================================
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 * ====================================================================
 *
 * This software consists of voluntary contributions made by many
 * individuals on behalf of the Apache Software Foundation.  For more
 * information on the Apache Software Foundation, please see
 * <http://www.apache.org/>.
 *
 */
// package org.apache.http.examples.client;
package testing01;

// my first version of a crawler, this program should
// 1. grab contents of a webpage
// 2. print out all text to a file

import java.io.*;
import java.util.ArrayList;
import java.util.List;

import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;

public class QuickStart {

    public static void main(String[] args) throws Exception {
        CloseableHttpClient httpclient = HttpClients.createDefault();
        String outputFile = "test.html";
        String baseWebSite = "http://www.ettoday.net/news/20130802/250478.htm";
        try {
            //HttpGet httpGet = new HttpGet(baseWebSite + "cat/politic/r");
            HttpGet httpGet = new HttpGet(baseWebSite);
            CloseableHttpResponse response1 = httpclient.execute(httpGet);
            // The underlying HTTP connection is still held by the response
            // object
            // to allow the response content to be streamed directly from the
            // network socket.
            // In order to ensure correct deallocation of system resources
            // the user MUST call CloseableHttpResponse#close() from a finally
            // clause.
            // Please note that if response content is not fully consumed the
            // underlying
            // connection cannot be safely re-used and will be shut down and
            // discarded
            // by the connection manager.
            try {
                System.out.println(response1.getStatusLine());
                HttpEntity entity1 = response1.getEntity();
                // do something useful with the response body
                // and ensure it is fully consumed
                String responseString = EntityUtils.toString(entity1, "UTF-8");
                // System.out.println(responseString);
                Writer out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outputFile), "UTF8"));
                out.write(responseString);
                out.close();
                EntityUtils.consume(entity1);
                System.out.println(baseWebSite + " output to " + outputFile + " successful");
            } catch (Exception e) {
                System.out.println(e.getMessage());
            } finally {
                response1.close();
            }
            /*
             * HttpPost httpPost = new HttpPost("http://targethost/login"); List
             * <NameValuePair> nvps = new ArrayList <NameValuePair>();
             * nvps.add(new BasicNameValuePair("username", "vip")); nvps.add(new
             * BasicNameValuePair("password", "secret")); httpPost.setEntity(new
             * UrlEncodedFormEntity(nvps)); CloseableHttpResponse response2 =
             * httpclient.execute(httpPost);
             * 
             * try { System.out.println(response2.getStatusLine()); HttpEntity
             * entity2 = response2.getEntity(); // do something useful with the
             * response body // and ensure it is fully consumed
             * EntityUtils.consume(entity2); } finally { response2.close(); }
             */
        } finally {
            httpclient.close();
        }
    }

}