Example usage for java.net HttpURLConnection getInputStream

List of usage examples for java.net HttpURLConnection getInputStream

  1. HOME
  2. Java
  3. java
  4. java.net.*
  5. HttpURLConnection
  6. getInputStream

Introduction

In this page you can find the example usage for java.net HttpURLConnection getInputStream.

Prototype

public InputStream getInputStream() throws IOException

Source Link

Document

Returns an input stream that reads from this open connection.

Usage

From source file:Main.java

public static void main(String args[]) throws Exception {
    URL url = new URL("http://www.google.com");
    HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();

    InputStream inStrm = httpCon.getInputStream();
    System.out.println("\nContent at " + url);
    int ch;//from   w  ww  . j ava2s . c o  m
    while (((ch = inStrm.read()) != -1))
        System.out.print((char) ch);
    inStrm.close();
}

From source file:Main.java

public static void main(String[] argv) throws Exception {
    byte[] b = new byte[1];
    Properties systemSettings = System.getProperties();
    systemSettings.put("http.proxyHost", "proxy.mydomain.local");
    systemSettings.put("http.proxyPort", "80");

    Authenticator.setDefault(new Authenticator() {
        protected PasswordAuthentication getPasswordAuthentication() {
            return new PasswordAuthentication("mydomain\\username", "password".toCharArray());
        }//from   w  w w  .j av  a2s .  c o  m
    });

    URL u = new URL("http://www.google.com");
    HttpURLConnection con = (HttpURLConnection) u.openConnection();
    DataInputStream di = new DataInputStream(con.getInputStream());
    while (-1 != di.read(b, 0, 1)) {
        System.out.print(new String(b));
    }
}

From source file:Main.java

public static void main(String[] argv) throws Exception {

    URL url = new URL("https://www.server.com");
    HttpURLConnection con = (HttpURLConnection) url.openConnection();

    BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
    String inputLine;// w w w.  j  a  va2  s  .  c om

    while ((inputLine = in.readLine()) != null)
        System.out.println(inputLine);
    in.close();
}

From source file:com.cloudera.earthquake.GetData.java

public static void main(String[] args) throws IOException {
    URL url = new URL("http://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_month.csv");
    HttpURLConnection conn = (HttpURLConnection) url.openConnection();
    conn.connect();/*from w  w  w .  ja v  a 2s  .co  m*/
    InputStream connStream = conn.getInputStream();

    FileSystem hdfs = FileSystem.get(new Configuration());
    FSDataOutputStream outStream = hdfs.create(new Path(args[0], "month.txt"));
    IOUtils.copy(connStream, outStream);

    outStream.close();
    connStream.close();
    conn.disconnect();
}

From source file:com.mycompany.test.Jaroop.java

/**
 * This is the main program which will receive the request, calls required methods
 * and processes the response.//from   w  w  w .j a v  a2s  .  c om
 * @param args
 * @throws IOException
 */
public static void main(String[] args) throws IOException {
    Scanner scannedInput = new Scanner(System.in);
    String in = "";
    if (args.length == 0) {
        System.out.println("Enter the query");
        in = scannedInput.nextLine();
    } else {
        in = args[0];
    }
    in = in.toLowerCase().replaceAll("\\s+", "_");
    int httpStatus = checkInvalidInput(in);
    if (httpStatus == 0) {
        System.out.print("Not found");
        System.exit(0);
    }
    String url = "https://en.wikipedia.org/w/api.php?format=json&action=query&prop=extracts&exintro=&explaintext=&titles="
            + in;
    HttpURLConnection connection = getConnection(url);
    BufferedReader input = new BufferedReader(new InputStreamReader(connection.getInputStream()));
    String request = "";
    StringBuilder response = new StringBuilder();
    while ((request = input.readLine()) != null) {
        //only appending what ever is required for JSON parsing and ignoring the rest
        response.append("{");
        //appending the key "extract" to the string so that the JSON parser can parse it's value,
        //also we don't need last 3 paranthesis in the response, excluding them as well.
        response.append(request.substring(request.indexOf("\"extract"), request.length() - 3));
    }
    parseJSON(response.toString());
}

From source file:Main.java

public static void main(String[] argv) throws Exception {
    URL url = new URL("http://www.java.com");
    URLConnection urlConnection = url.openConnection();
    HttpURLConnection connection = null;
    if (urlConnection instanceof HttpURLConnection) {
        connection = (HttpURLConnection) urlConnection;
    } else {//w w w.j  av a 2s.  c  om
        System.out.println("Please enter an HTTP URL.");
        return;
    }
    BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
    String urlString = "";
    String current;
    while ((current = in.readLine()) != null) {
        urlString += current;
    }
    System.out.println(urlString);
}

From source file:com.dict.crawl.NewsNationalGeographicCrawler.java

public static void main(String[] args) throws Exception {
    /*string,crawlPath??crawlPath,
      ????crawlPath//from  w ww.j a  v a2 s.  co  m
    */

    NewsNationalGeographicCrawler crawler = new NewsNationalGeographicCrawler("data/NewsNationalGeographic");
    crawler.setThreads(2);
    crawler.addSeed("http://ngm.nationalgeographic.com/");

    if (BaseCrawler.isNormalTime()) {

        crawler.addSeed("http://ngm.nationalgeographic.com/archives");
        crawler.addSeed("http://ngm.nationalgeographic.com/featurehub");
        //
        //}

        String jsonUrl = "http://news.nationalgeographic.com/bin/services/news/public/query/content.json?pageSize=20&page=0&contentTypes=news/components/pagetypes/article,news/components/pagetypes/simple-article,news/components/pagetypes/photo-gallery";

        URL urls = new URL(jsonUrl);
        HttpURLConnection urlConnection = (HttpURLConnection) urls.openConnection();
        InputStream is = urlConnection.getInputStream();
        Reader rd = new InputStreamReader(is, "utf-8");
        JsonArray json = new JsonParser().parse(rd).getAsJsonArray();
        for (JsonElement jOb : json) {
            String url = jOb.getAsJsonObject().get("page").getAsJsonObject().get("url").getAsString();
            if (url != null && !url.equals(""))
                crawler.addSeed(url);
        }

    }

    //        crawler.addSeed("http://news.nationalgeographic.com/2016/01/160118-mummies-world-bog-egypt-science/");
    //        List<Map<String, Object>> urls = crawler.jdbcTemplate.queryForList("SELECT id,title,url FROM parser_page where host like '%news.national%' ORDER by id desc;");
    //        for(int i = 0; i < urls.size(); i++) {
    //            String url = (String) urls.get(i).get("url");
    //            String title = (String) urls.get(i).get("title");
    ////            int id = (int) urls.get(i).get("id");
    //            crawler.addSeed(url);
    //        }

    //        Config
    Config.WAIT_THREAD_END_TIME = 1000 * 60 * 5;//???kill
    //        Config.TIMEOUT_CONNECT = 1000*10;
    //        Config.TIMEOUT_READ = 1000*30;
    Config.requestMaxInterval = 1000 * 60 * 20;//??-?>hung

    //requester??http??requester?http/socks?
    HttpRequesterImpl requester = (HttpRequesterImpl) crawler.getHttpRequester();
    AntiAntiSpiderHelper.defaultUserAgent(requester);

    //        requester.setUserAgent("Mozilla/5.0 (X11; Linux i686; rv:33.0) Gecko/20100101 Firefox/33.0");
    //        requester.setCookie("CNZZDATA1950488=cnzz_eid%3D739324831-1432460954-null%26ntime%3D1432460954; wdcid=44349d3f2aa96e51; vjuids=-53d395da8.14eca7eed44.0.f17be67e; CNZZDATA3473518=cnzz_eid%3D1882396923-1437965756-%26ntime%3D1440635510; pt_37a49e8b=uid=FuI4KYEfVz5xq7L4nzPd1w&nid=1&vid=r4AhSBmxisCiyeolr3V2Ow&vn=1&pvn=1&sact=1440639037916&to_flag=0&pl=t4NrgYqSK5M357L2nGEQCw*pt*1440639015734; _ga=GA1.3.1121158748.1437970841; __auc=c00a6ac114d85945f01d9c30128; CNZZDATA1975683=cnzz_eid%3D250014133-1432460541-null%26ntime%3D1440733997; CNZZDATA1254041250=2000695407-1442220871-%7C1442306691; pt_7f0a67e8=uid=6lmgYeZ3/jSObRMeK-t27A&nid=0&vid=lEKvEtZyZdd0UC264UyZnQ&vn=2&pvn=1&sact=1442306703728&to_flag=0&pl=7GB3sYS/PJDo1mY0qeu2cA*pt*1442306703728; 7NSx_98ef_saltkey=P05gN8zn; 7NSx_98ef_lastvisit=1444281282; IframeBodyHeight=256; NTVq_98ef_saltkey=j5PydYru; NTVq_98ef_lastvisit=1444282735; NTVq_98ef_atarget=1; NTVq_98ef_lastact=1444286377%09api.php%09js; 7NSx_98ef_sid=hZyDwc; __utmt=1; __utma=155578217.1121158748.1437970841.1443159326.1444285109.23; __utmb=155578217.57.10.1444285109; __utmc=155578217; __utmz=155578217.1439345650.3.2.utmcsr=travel.chinadaily.com.cn|utmccn=(referral)|utmcmd=referral|utmcct=/; CNZZDATA3089622=cnzz_eid%3D1722311508-1437912344-%26ntime%3D1444286009; wdlast=1444287704; vjlast=1437916393.1444285111.11; 7NSx_98ef_lastact=1444287477%09api.php%09chinadaily; pt_s_3bfec6ad=vt=1444287704638&cad=; pt_3bfec6ad=uid=bo87MAT/HC3hy12HDkBg1A&nid=0&vid=erwHQyFKxvwHXYc4-r6n-w&vn=28&pvn=2&sact=1444287708079&to_flag=0&pl=kkgvLoEHXsCD2gs4VJaWQg*pt*1444287704638; pt_t_3bfec6ad=?id=3bfec6ad.bo87MAT/HC3hy12HDkBg1A.erwHQyFKxvwHXYc4-r6n-w.kkgvLoEHXsCD2gs4VJaWQg.nZJ9Aj/bgfNDIKBXI5TwRQ&stat=167.132.1050.1076.1body%20div%3Aeq%288%29%20ul%3Aeq%280%29%20a%3Aeq%282%29.0.0.1595.3441.146.118&ptif=4");
    //?? Mozilla/5.0 (X11; Linux i686; rv:34.0) Gecko/20100101 Firefox/34.0
    //c requester.setProxy("
    /*
            
    //??
    RandomProxyGenerator proxyGenerator=new RandomProxyGenerator();
    proxyGenerator.addProxy("127.0.0.1",8080,Proxy.Type.SOCKS);
    requester.setProxyGenerator(proxyGenerator);
    */

    /*??*/
    //        crawler.setResumable(true);
    crawler.setResumable(false);

    crawler.start(2);
}

From source file:Main.java

public static void main(String[] argv) throws Exception {
    byte[] b = new byte[1];
    Properties systemSettings = System.getProperties();
    systemSettings.put("http.proxyHost", "proxy.mydomain.local");
    systemSettings.put("http.proxyPort", "80");

    URL u = new URL("http://www.google.com");
    HttpURLConnection con = (HttpURLConnection) u.openConnection();
    BASE64Encoder encoder = new BASE64Encoder();
    String encodedUserPwd = encoder.encode("mydomain\\MYUSER:MYPASSWORD".getBytes());
    con.setRequestProperty("Proxy-Authorization", "Basic " + encodedUserPwd);
    DataInputStream di = new DataInputStream(con.getInputStream());
    while (-1 != di.read(b, 0, 1)) {
        System.out.print(new String(b));
    }// ww  w. j  a va 2 s  . c om
}

From source file:icevaluation.BingAPIAccess.java

public static void main(String[] args) {
    String searchText = "arts site:wikipedia.org";
    searchText = searchText.replaceAll(" ", "%20");
    // String accountKey="jTRIJt9d8DR2QT/Z3BJCAvY1BfoXj0zRYgSZ8deqHHo";
    String accountKey = "JfeJSA3x6CtsyVai0+KEP0A6CYEUBT8VWhZmm9CS738";

    byte[] accountKeyBytes = Base64.encodeBase64((accountKey + ":" + accountKey).getBytes());
    String accountKeyEnc = new String(accountKeyBytes);
    URL url;/*w  w  w  . ja v a  2  s . c  om*/
    try {
        url = new URL("https://api.datamarket.azure.com/Bing/Search/v1/Composite?Sources=%27Web%27&Query=%27"
                + searchText + "%27&$format=JSON");
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setRequestMethod("GET");
        conn.setRequestProperty("Authorization", "Basic " + accountKeyEnc);

        BufferedReader br = new BufferedReader(new InputStreamReader((conn.getInputStream())));
        StringBuilder sb = new StringBuilder();
        String output;
        System.out.println("Output from Server .... \n");
        //write json to string sb
        int c = 0;
        if ((output = br.readLine()) != null) {
            System.out.println("Output is: " + output);
            sb.append(output);
            c++;
            //System.out.println("C:"+c);

        }

        conn.disconnect();
        //find webtotal among output      
        int find = sb.indexOf("\"WebTotal\":\"");
        int startindex = find + 12;
        System.out.println("Find: " + find);

        int lastindex = sb.indexOf("\",\"WebOffset\"");

        System.out.println(sb.substring(startindex, lastindex));

    } catch (MalformedURLException e1) {
        e1.printStackTrace();
    } catch (IOException e) {

        e.printStackTrace();
    }

}

From source file:JSON.JasonJSON.java

public static void main(String[] args) throws Exception {

    URL Url = new URL("http://api.wunderground.com/api/22b4347c464f868e/conditions/q/Colorado/COS.json");
    //This next URL is still being played with. Some of the formatting is hard to figure out, but Wunderground 
    //writes a perfectly formatted JSON file that is easy to read with Java.
    // URL Url = new URL("https://api.darksky.net/forecast/08959bb1e2c7eae0f3d1fafb5d538032/38.886,-104.7201");

    try {//  ww  w  .  j  a  v a 2s  . c  om

        HttpURLConnection urlCon = (HttpURLConnection) Url.openConnection();

        //          This part will read the data returned thru HTTP and load it into memory
        //          I have this code left over from my CIT260 project.
        InputStream stream = urlCon.getInputStream();
        BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
        StringBuilder result = new StringBuilder();
        String line;
        while ((line = reader.readLine()) != null) {
            result.append(line);
        }

        // The next lines read certain parts of the JSON data and print it out on the screen
        //Creates the JSONObject object and loads the JSON file from the URLConnection
        //Into a StringWriter object. I am printing this out in raw format just so I can see it doing something

        JSONObject json = new JSONObject(result.toString());

        JSONObject coloradoInfo = (JSONObject) json.get("current_observation");

        StringWriter out = new StringWriter();
        json.write(out);
        String jsonTxt = json.toString();
        System.out.print(jsonTxt);

        List<String> list = new ArrayList<>();
        JSONArray array = json.getJSONArray(jsonTxt);
        System.out.print(jsonTxt);

        // for (int i =0;i<array.length();i++){
        //list.add(array.getJSONObject(i).getString("current_observation"));
        //}

        String wunderGround = "Data downloaded from: " +

                coloradoInfo.getJSONObject("image").getString("title") + "\nLink\t\t: "
                + coloradoInfo.getJSONObject("image").getString("link") + "\nCity\t\t: "
                + coloradoInfo.getJSONObject("display_location").getString("city") + "\nState\t\t: "
                + coloradoInfo.getJSONObject("display_location").getString("state_name") + "\nTime\t\t: "
                + coloradoInfo.get("observation_time_rfc822") + "\nTemperature\t\t: "
                + coloradoInfo.get("temperature_string") + "\nWindchill\t\t: "
                + coloradoInfo.get("windchill_string") + "\nRelative Humidity\t: "
                + coloradoInfo.get("relative_humidity") + "\nWind\t\t\t: " + coloradoInfo.get("wind_string")
                + "\nWind Direction\t\t: " + coloradoInfo.get("wind_dir") + "\nBarometer Pressure\t\t: "
                + coloradoInfo.get("pressure_in");

        System.out.println("\nColorado Springs Weather:");
        System.out.println("____________________________________");
        System.out.println(wunderGround);

    } catch (IOException e) {
        System.out.println("***ERROR*******************ERROR********************. " + "\nURL: " + Url.toString()
                + "\nERROR: " + e.toString());
    }

}