Example usage for java.net URL openConnection

List of usage examples for java.net URL openConnection

  1. HOME
  2. Java
  3. java
  4. java.net.*
  5. URL
  6. openConnection

Introduction

In this page you can find the example usage for java.net URL openConnection.

Prototype

public URLConnection openConnection() throws java.io.IOException

Source Link

Document

Returns a java.net.URLConnection URLConnection instance that represents a connection to the remote object referred to by the URL .

Usage

From source file:com.pp.dcasajus.forkpoint.GcmSender.java

public static void main(String[] args) {
    if (args.length < 1 || args.length > 2 || args[0] == null) {
        System.err.println("usage: ./gradlew run -Pmsg=\"MESSAGE\" [-Pto=\"DEVICE_TOKEN\"]");
        System.err.println("");
        System.err.println(//from   ww  w .j ava  2 s . co m
                "Specify a test message to broadcast via GCM. If a device's GCM registration token is\n"
                        + "specified, the message will only be sent to that device. Otherwise, the message \n"
                        + "will be sent to all devices subscribed to the \"global\" topic.");
        System.err.println("");
        System.err.println(
                "Example (Broadcast):\n" + "On Windows:   .\\gradlew.bat run -Pmsg=\"<Your_Message>\"\n"
                        + "On Linux/Mac: ./gradlew run -Pmsg=\"<Your_Message>\"");
        System.err.println("");
        System.err.println("Example (Unicast):\n"
                + "On Windows:   .\\gradlew.bat run -Pmsg=\"<Your_Message>\" -Pto=\"<Your_Token>\"\n"
                + "On Linux/Mac: ./gradlew run -Pmsg=\"<Your_Message>\" -Pto=\"<Your_Token>\"");
        System.exit(1);
    }
    try {
        // Prepare JSON containing the GCM message content. What to send and where to send.
        JSONObject jGcmData = new JSONObject();
        JSONObject jData = new JSONObject();
        jData.put("message", args[0].trim());
        // Where to send GCM message.
        if (args.length > 1 && args[1] != null) {
            jGcmData.put("to", args[1].trim());
        } else {
            jGcmData.put("to", "/topics/global");
        }
        // What to send in GCM message.
        jGcmData.put("data", jData);

        // Create connection to send GCM Message request.
        URL url = new URL("https://android.googleapis.com/gcm/send");
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setRequestProperty("Authorization", "key=" + API_KEY);
        conn.setRequestProperty("Content-Type", "application/json");
        conn.setRequestMethod("POST");
        conn.setDoOutput(true);

        // Send GCM message content.
        OutputStream outputStream = conn.getOutputStream();
        outputStream.write(jGcmData.toString().getBytes());

        // Read GCM response.
        InputStream inputStream = conn.getInputStream();
        String resp = IOUtils.toString(inputStream);
        System.out.println(resp);
        System.out.println("Check your device/emulator for notification or logcat for "
                + "confirmation of the receipt of the GCM message.");
    } catch (IOException e) {
        System.out.println("Unable to send GCM message.");
        System.out.println("Please ensure that API_KEY has been replaced by the server "
                + "API key, and that the device's registration token is correct (if specified).");
        e.printStackTrace();
    } catch (JSONException e) {
        e.printStackTrace();
    }
}

From source file:URLConnectionTest.java

public static void main(String[] args) {
    try {//from   w ww  . jav  a  2 s.c o  m
        String urlName;
        if (args.length > 0)
            urlName = args[0];
        else
            urlName = "http://java.sun.com";

        URL url = new URL(urlName);
        URLConnection connection = url.openConnection();

        // set username, password if specified on command line

        if (args.length > 2) {
            String username = args[1];
            String password = args[2];
            String input = username + ":" + password;
            String encoding = base64Encode(input);
            connection.setRequestProperty("Authorization", "Basic " + encoding);
        }

        connection.connect();

        // print header fields

        Map<String, List<String>> headers = connection.getHeaderFields();
        for (Map.Entry<String, List<String>> entry : headers.entrySet()) {
            String key = entry.getKey();
            for (String value : entry.getValue())
                System.out.println(key + ": " + value);
        }

        // print convenience functions

        System.out.println("----------");
        System.out.println("getContentType: " + connection.getContentType());
        System.out.println("getContentLength: " + connection.getContentLength());
        System.out.println("getContentEncoding: " + connection.getContentEncoding());
        System.out.println("getDate: " + connection.getDate());
        System.out.println("getExpiration: " + connection.getExpiration());
        System.out.println("getLastModifed: " + connection.getLastModified());
        System.out.println("----------");

        Scanner in = new Scanner(connection.getInputStream());

        // print first ten lines of contents

        for (int n = 1; in.hasNextLine() && n <= 10; n++)
            System.out.println(in.nextLine());
        if (in.hasNextLine())
            System.out.println(". . .");
    } catch (IOException e) {
        e.printStackTrace();
    }
}

From source file:com.jixstreet.temanusaha.gcm.GcmSender.java

public static void main(String[] args) {
    if (args.length < 1 || args.length > 2 || args[0] == null) {
        System.err.println("usage: ./gradlew run -Pmsg=\"MESSAGE\" [-Pto=\"DEVICE_TOKEN\"]");
        System.err.println("");
        System.err.println(/*from www .j a v  a2  s.  c o  m*/
                "Specify a test message to broadcast via GCM. If a device's GCM registration token is\n"
                        + "specified, the message will only be sent to that device. Otherwise, the message \n"
                        + "will be sent to all devices subscribed to the \"global\" topic.");
        System.err.println("");
        System.err.println(
                "Example (Broadcast):\n" + "On Windows:   .\\gradlew.bat run -Pmsg=\"<Your_Message>\"\n"
                        + "On Linux/Mac: ./gradlew run -Pmsg=\"<Your_Message>\"");
        System.err.println("");
        System.err.println("Example (Unicast):\n"
                + "On Windows:   .\\gradlew.bat run -Pmsg=\"<Your_Message>\" -Pto=\"<Your_Token>\"\n"
                + "On Linux/Mac: ./gradlew run -Pmsg=\"<Your_Message>\" -Pto=\"<Your_Token>\"");
        System.exit(1);
    }
    try {
        // Prepare JSON containing the GCM message content. What to send and where to send.
        JSONObject jGcmData = new JSONObject();
        JSONObject jData = new JSONObject();
        try {
            jData.put("message", args[0].trim());
        } catch (JSONException e) {
            e.printStackTrace();
        }
        // Where to send GCM message.
        if (args.length > 1 && args[1] != null) {
            try {
                jGcmData.put("to", args[1].trim());
            } catch (JSONException e) {
                e.printStackTrace();
            }
        } else {
            try {
                jGcmData.put("to", "/topics/global");
            } catch (JSONException e) {
                e.printStackTrace();
            }
        }
        // What to send in GCM message.
        try {
            JSONObject data = jGcmData.put("data", jData);
        } catch (JSONException e) {
            e.printStackTrace();
        }

        // Create connection to send GCM Message request.
        URL url = new URL("https://android.googleapis.com/gcm/send");
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setRequestProperty("Authorization", "key=" + API_KEY);
        conn.setRequestProperty("Content-Type", "application/json");
        conn.setRequestMethod("POST");
        conn.setDoOutput(true);

        // Send GCM message content.
        OutputStream outputStream = conn.getOutputStream();
        outputStream.write(jGcmData.toString().getBytes());

        // Read GCM response.
        InputStream inputStream = conn.getInputStream();
        String resp = IOUtils.toString(inputStream);
        System.out.println(resp);
        System.out.println("Check your device/emulator for notification or logcat for "
                + "confirmation of the receipt of the GCM message.");
    } catch (IOException e) {
        System.out.println("Unable to send GCM message.");
        System.out.println("Please ensure that API_KEY has been replaced by the server "
                + "API key, and that the device's registration token is correct (if specified).");
        e.printStackTrace();
    }
}

From source file:com.newproject.ApacheHttp.java

public static void main(String[] args) throws Exception {
    CloseableHttpClient httpclient = HttpClients.createDefault();
    String jsonFilePath = "/Users/vikasmohandoss/Documents/Cloud/test.txt";
    String url = "http://www.sentiment140.com/api/bulkClassifyJson&appid=vm2446@columbia.edu";
    JSONParser jsonParser = new JSONParser();
    JSONObject jsonObject = new JSONObject();
    URL obj = new URL(url);
    HttpURLConnection con = (HttpURLConnection) obj.openConnection();
    try {/*  www.ja  va 2 s.  c  o  m*/
        FileReader fileReader = new FileReader(jsonFilePath);
        jsonObject = (JSONObject) jsonParser.parse(fileReader);
    } catch (FileNotFoundException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    } catch (ParseException e) {
        e.printStackTrace();
    }
    System.out.println(jsonObject.toString());
    /*try {
    /*HttpGet httpGet = new HttpGet("http://httpbin.org/get");
    CloseableHttpResponse response1 = httpclient.execute(httpGet);
    // The underlying HTTP connection is still held by the response object
    // to allow the response content to be streamed directly from the network socket.
    // In order to ensure correct deallocation of system resources
    // the user MUST call CloseableHttpResponse#close() from a finally clause.
    // Please note that if response content is not fully consumed the underlying
    // connection cannot be safely re-used and will be shut down and discarded
    // by the connection manager.
    try {
        System.out.println(response1.getStatusLine());
        HttpEntity entity1 = response1.getEntity();
        // do something useful with the response body
        // and ensure it is fully consumed
        EntityUtils.consume(entity1);
    } finally {
        response1.close();
    }
    HttpPost httpPost = new HttpPost("http://httpbin.org/post");
    List <NameValuePair> nvps = new ArrayList <NameValuePair>();
    nvps.add(new BasicNameValuePair("username", "vip"));
    nvps.add(new BasicNameValuePair("password", "secret"));
    httpPost.setEntity(new UrlEncodedFormEntity(nvps));
    CloseableHttpResponse response2 = httpclient.execute(httpPost);
            
    try {
        System.out.println(response2.getStatusLine());
        HttpEntity entity2 = response2.getEntity();
        // do something useful with the response body
        // and ensure it is fully consumed
        EntityUtils.consume(entity2);
    } finally {
        response2.close();
    }
    } finally {
    httpclient.close();
    }*/
    try {
        HttpPost request = new HttpPost("http://www.sentiment140.com/api/bulkClassifyJson");
        StringEntity params = new StringEntity(jsonObject.toString());
        request.addHeader("content-type", "application/json");
        request.setEntity(params);
        HttpResponse response = httpclient.execute(request);
        System.out.println(response.toString());
        String result = EntityUtils.toString(response.getEntity());
        System.out.println(result);
        try {
            File file = new File("/Users/vikasmohandoss/Documents/Cloud/sentiment.txt");
            // if file doesnt exists, then create it
            if (!file.exists()) {
                file.createNewFile();
            }
            FileWriter fw = new FileWriter(file.getAbsoluteFile());
            BufferedWriter bw = new BufferedWriter(fw);
            bw.write(result);
            bw.close();
            System.out.println("Done");
        } catch (IOException e) {
            e.printStackTrace();
        }
        // handle response here...
    } catch (Exception ex) {
        // handle exception here
    } finally {
        httpclient.close();
    }
}

From source file:com.yahoo.athenz.example.ztoken.HttpExampleClient.java

public static void main(String[] args) throws MalformedURLException, IOException {

    // parse our command line to retrieve required input

    CommandLine cmd = parseCommandLine(args);

    String domainName = cmd.getOptionValue("domain");
    String serviceName = cmd.getOptionValue("service");
    String privateKeyPath = cmd.getOptionValue("pkey");
    String keyId = cmd.getOptionValue("keyid");
    String url = cmd.getOptionValue("url");
    String ztsUrl = cmd.getOptionValue("ztsurl");
    String providerDomain = cmd.getOptionValue("provider-domain");
    String providerRole = cmd.getOptionValue("provider-role");

    // we need to generate our principal credentials (ntoken). In
    // addition to the domain and service names, we need the
    // the service's private key and the key identifier - the
    // service with the corresponding public key must already be
    // registered in ZMS

    PrivateKey privateKey = Crypto.loadPrivateKey(new File(privateKeyPath));
    ServiceIdentityProvider identityProvider = new SimpleServiceIdentityProvider(domainName, serviceName,
            privateKey, keyId);/*from  w w w.j ava2  s.c o  m*/

    // now we need to retrieve a role token (ztoken) for accessing
    // the provider Athenz enabled service

    RoleToken roleToken = null;
    try (ZTSClient ztsClient = new ZTSClient(ztsUrl, domainName, serviceName, identityProvider)) {
        roleToken = ztsClient.getRoleToken(providerDomain, providerRole);
    }

    if (roleToken == null) {
        System.out.println(
                "Unable to retrieve role token for: " + providerRole + " in domain: " + providerDomain);
        System.exit(1);
    }

    URL obj = new URL(url);
    HttpURLConnection con = (HttpURLConnection) obj.openConnection();

    // set our Athenz credentials. The ZTSClient provides the header
    // name that we must use for authorization token while the role
    // token itself provides the token string (ztoken).

    System.out.println("Using RoleToken: " + roleToken.getToken());
    con.setRequestProperty(ZTSClient.getHeader(), roleToken.getToken());

    // now process our request

    int responseCode = con.getResponseCode();
    switch (responseCode) {
    case HttpURLConnection.HTTP_FORBIDDEN:
        System.out.println("Request was forbidden - not authorized: " + con.getResponseMessage());
        break;
    case HttpURLConnection.HTTP_OK:
        System.out.println("Successful response: ");
        try (BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()))) {
            String inputLine;
            while ((inputLine = in.readLine()) != null) {
                System.out.println(inputLine);
            }
        }
        break;
    default:
        System.out.println("Request failed - response status code: " + responseCode);
    }
}

From source file:SifterJava.java

public static void main(String[] args) throws Exception {
    /** Enter your sifter account and access key in the 
    ** string fields below *///from  ww  w  .  j a v  a2  s .  com

    // create URL object to SifterAPI
    String yourAccount = "your-account"; // enter your account name
    String yourDomain = yourAccount + ".sifterapp.com";
    URL sifter = new URL("https", yourDomain, "/api/projects");

    // open connectino to SifterAPI
    URLConnection sifterConnection = sifter.openConnection();

    // add Access Key to header request
    String yourAccessKey = "your-32char-sifterapi-access-key"; // enter your access key
    sifterConnection.setRequestProperty("X-Sifter-Token", yourAccessKey);

    // add Accept: application/json to header - also not necessary
    sifterConnection.addRequestProperty("Accept", "application/json");

    // URLconnection.connect() not necessary
    // getInputStream will connect

    // don't make a content handler, org.json reads a stream!

    // create buffer and open input stream
    BufferedReader in = new BufferedReader(new InputStreamReader(sifterConnection.getInputStream()));
    // don't readline, create stringbuilder, append, create string

    // construct json tokener from input stream or buffered reader
    JSONTokener x = new JSONTokener(in);

    // initialize "projects" JSONObject from string
    JSONObject projects = new JSONObject(x);

    // prettyprint "projects"
    System.out.println("************ projects ************");
    System.out.println(projects.toString(2));

    // array of projects
    JSONArray array = projects.getJSONArray("projects");
    int arrayLength = array.length();
    JSONObject[] p = new JSONObject[arrayLength];

    // projects
    for (int i = 0; i < arrayLength; i++) {
        p[i] = array.getJSONObject(i);
        System.out.println("************ project: " + (i + 1) + " ************");
        System.out.println(p[i].toString(2));
    }

    // check field names
    String[] checkNames = { "api_url", "archived", "api_issues_url", "milestones_url", "api_milestones_url",
            "api_categories_url", "issues_url", "name", "url", "api_people_url", "primary_company_name" };

    // project field names
    String[] fieldNames = JSONObject.getNames(p[0]);
    int numKeys = p[0].length();
    SifterProj[] proj = new SifterProj[arrayLength];

    for (int j = 0; j < arrayLength; j++) {
        proj[j] = new SifterProj(p[j].get("api_url").toString(), p[j].get("archived").toString(),
                p[j].get("api_issues_url").toString(), p[j].get("milestones_url").toString(),
                p[j].get("api_milestones_url").toString(), p[j].get("api_categories_url").toString(),
                p[j].get("issues_url").toString(), p[j].get("name").toString(), p[j].get("url").toString(),
                p[j].get("api_people_url").toString(), p[j].get("primary_company_name").toString());
        System.out.println("************ project: " + (j + 1) + " ************");
        for (int i = 0; i < numKeys; i++) {
            System.out.print(fieldNames[i]);
            System.out.print(" : ");
            System.out.println(p[j].get(fieldNames[i]));
        }
    }
}

From source file:OldExtractor.java

/**
 * @param args the command line arguments
 */// ww w.jav  a 2  s.c om
public static void main(String[] args) {
    // TODO code application logic here
    String bingUrl = "https://api.datamarket.azure.com/Bing/Search/Web?$top=10&$format=Atom&Query=%27gates%27";
    //Provide your account key here.
    String accountKey = "ghTYY7wD6LpyxUO9VRR7e1f98WFhHWYERMcw87aQTqQ";
    //  String accountKey = "xqbCjT87/MQz25JWdRzgMHdPkGYnOz77IYmP5FUIgC8";

    byte[] accountKeyBytes = Base64.encodeBase64((accountKey + ":" + accountKey).getBytes());
    String accountKeyEnc = new String(accountKeyBytes);

    try {
        URL url = new URL(bingUrl);
        URLConnection urlConnection = url.openConnection();

        urlConnection.setRequestProperty("Authorization", "Basic " + accountKeyEnc);
        InputStream inputStream = (InputStream) urlConnection.getContent();
        byte[] contentRaw = new byte[urlConnection.getContentLength()];
        inputStream.read(contentRaw);
        String content = new String(contentRaw);
        //System.out.println(content);
        try {
            File file = new File("Results.xml");
            FileWriter fileWriter = new FileWriter(file);
            fileWriter.write(content);
            //fileWriter.write("a test");
            fileWriter.flush();
            fileWriter.close();

        } catch (IOException e) {
            System.out.println(e);
        }

        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();

        try {
            //System.out.println("here");
            //Using factory get an instance of document builder
            DocumentBuilder db = dbf.newDocumentBuilder();

            //parse using builder to get DOM representation of the XML file
            Document dom = db.parse("Results.xml");
            Element docEle = (Element) dom.getDocumentElement();

            //get a nodelist of elements

            NodeList nl = docEle.getElementsByTagName("d:Url");
            if (nl != null && nl.getLength() > 0) {
                for (int i = 0; i < nl.getLength(); i++) {

                    //get the employee element
                    Element el = (Element) nl.item(i);
                    // System.out.println("here");
                    System.out.println(el.getTextContent());

                    //get the Employee object
                    //Employee e = getEmployee(el);

                    //add it to list
                    //myEmpls.add(e);
                }
            }
            NodeList n2 = docEle.getElementsByTagName("d:Title");
            if (n2 != null && n2.getLength() > 0) {
                for (int i = 0; i < n2.getLength(); i++) {

                    //get the employee element
                    Element e2 = (Element) n2.item(i);
                    // System.out.println("here");
                    System.out.println(e2.getTextContent());

                    //get the Employee object
                    //Employee e = getEmployee(el);

                    //add it to list
                    //myEmpls.add(e);
                }
            }

            NodeList n3 = docEle.getElementsByTagName("d:Description");
            if (n3 != null && n3.getLength() > 0) {
                for (int i = 0; i < n3.getLength(); i++) {

                    //get the employee element
                    Element e3 = (Element) n3.item(i);
                    // System.out.println("here");
                    System.out.println(e3.getTextContent());

                    //get the Employee object
                    //Employee e = getEmployee(el);

                    //add it to list
                    //myEmpls.add(e);
                }
            }

        } catch (SAXException se) {
            se.printStackTrace();
        } catch (ParserConfigurationException pe) {
            pe.printStackTrace();
        } catch (IOException ioe) {
            ioe.printStackTrace();
        }

    } catch (IOException e) {
        System.out.println(e);
    }

    //The content string is the xml/json output from Bing.

}

From source file:com.dict.crawl.NewsNationalGeographicCrawler.java

public static void main(String[] args) throws Exception {
    /*string,crawlPath??crawlPath,
      ????crawlPath/*w w w.  j  a va2 s.  c o  m*/
    */

    NewsNationalGeographicCrawler crawler = new NewsNationalGeographicCrawler("data/NewsNationalGeographic");
    crawler.setThreads(2);
    crawler.addSeed("http://ngm.nationalgeographic.com/");

    if (BaseCrawler.isNormalTime()) {

        crawler.addSeed("http://ngm.nationalgeographic.com/archives");
        crawler.addSeed("http://ngm.nationalgeographic.com/featurehub");
        //
        //}

        String jsonUrl = "http://news.nationalgeographic.com/bin/services/news/public/query/content.json?pageSize=20&page=0&contentTypes=news/components/pagetypes/article,news/components/pagetypes/simple-article,news/components/pagetypes/photo-gallery";

        URL urls = new URL(jsonUrl);
        HttpURLConnection urlConnection = (HttpURLConnection) urls.openConnection();
        InputStream is = urlConnection.getInputStream();
        Reader rd = new InputStreamReader(is, "utf-8");
        JsonArray json = new JsonParser().parse(rd).getAsJsonArray();
        for (JsonElement jOb : json) {
            String url = jOb.getAsJsonObject().get("page").getAsJsonObject().get("url").getAsString();
            if (url != null && !url.equals(""))
                crawler.addSeed(url);
        }

    }

    //        crawler.addSeed("http://news.nationalgeographic.com/2016/01/160118-mummies-world-bog-egypt-science/");
    //        List<Map<String, Object>> urls = crawler.jdbcTemplate.queryForList("SELECT id,title,url FROM parser_page where host like '%news.national%' ORDER by id desc;");
    //        for(int i = 0; i < urls.size(); i++) {
    //            String url = (String) urls.get(i).get("url");
    //            String title = (String) urls.get(i).get("title");
    ////            int id = (int) urls.get(i).get("id");
    //            crawler.addSeed(url);
    //        }

    //        Config
    Config.WAIT_THREAD_END_TIME = 1000 * 60 * 5;//???kill
    //        Config.TIMEOUT_CONNECT = 1000*10;
    //        Config.TIMEOUT_READ = 1000*30;
    Config.requestMaxInterval = 1000 * 60 * 20;//??-?>hung

    //requester??http??requester?http/socks?
    HttpRequesterImpl requester = (HttpRequesterImpl) crawler.getHttpRequester();
    AntiAntiSpiderHelper.defaultUserAgent(requester);

    //        requester.setUserAgent("Mozilla/5.0 (X11; Linux i686; rv:33.0) Gecko/20100101 Firefox/33.0");
    //        requester.setCookie("CNZZDATA1950488=cnzz_eid%3D739324831-1432460954-null%26ntime%3D1432460954; wdcid=44349d3f2aa96e51; vjuids=-53d395da8.14eca7eed44.0.f17be67e; CNZZDATA3473518=cnzz_eid%3D1882396923-1437965756-%26ntime%3D1440635510; pt_37a49e8b=uid=FuI4KYEfVz5xq7L4nzPd1w&nid=1&vid=r4AhSBmxisCiyeolr3V2Ow&vn=1&pvn=1&sact=1440639037916&to_flag=0&pl=t4NrgYqSK5M357L2nGEQCw*pt*1440639015734; _ga=GA1.3.1121158748.1437970841; __auc=c00a6ac114d85945f01d9c30128; CNZZDATA1975683=cnzz_eid%3D250014133-1432460541-null%26ntime%3D1440733997; CNZZDATA1254041250=2000695407-1442220871-%7C1442306691; pt_7f0a67e8=uid=6lmgYeZ3/jSObRMeK-t27A&nid=0&vid=lEKvEtZyZdd0UC264UyZnQ&vn=2&pvn=1&sact=1442306703728&to_flag=0&pl=7GB3sYS/PJDo1mY0qeu2cA*pt*1442306703728; 7NSx_98ef_saltkey=P05gN8zn; 7NSx_98ef_lastvisit=1444281282; IframeBodyHeight=256; NTVq_98ef_saltkey=j5PydYru; NTVq_98ef_lastvisit=1444282735; NTVq_98ef_atarget=1; NTVq_98ef_lastact=1444286377%09api.php%09js; 7NSx_98ef_sid=hZyDwc; __utmt=1; __utma=155578217.1121158748.1437970841.1443159326.1444285109.23; __utmb=155578217.57.10.1444285109; __utmc=155578217; __utmz=155578217.1439345650.3.2.utmcsr=travel.chinadaily.com.cn|utmccn=(referral)|utmcmd=referral|utmcct=/; CNZZDATA3089622=cnzz_eid%3D1722311508-1437912344-%26ntime%3D1444286009; wdlast=1444287704; vjlast=1437916393.1444285111.11; 7NSx_98ef_lastact=1444287477%09api.php%09chinadaily; pt_s_3bfec6ad=vt=1444287704638&cad=; pt_3bfec6ad=uid=bo87MAT/HC3hy12HDkBg1A&nid=0&vid=erwHQyFKxvwHXYc4-r6n-w&vn=28&pvn=2&sact=1444287708079&to_flag=0&pl=kkgvLoEHXsCD2gs4VJaWQg*pt*1444287704638; pt_t_3bfec6ad=?id=3bfec6ad.bo87MAT/HC3hy12HDkBg1A.erwHQyFKxvwHXYc4-r6n-w.kkgvLoEHXsCD2gs4VJaWQg.nZJ9Aj/bgfNDIKBXI5TwRQ&stat=167.132.1050.1076.1body%20div%3Aeq%288%29%20ul%3Aeq%280%29%20a%3Aeq%282%29.0.0.1595.3441.146.118&ptif=4");
    //?? Mozilla/5.0 (X11; Linux i686; rv:34.0) Gecko/20100101 Firefox/34.0
    //c requester.setProxy("
    /*
            
    //??
    RandomProxyGenerator proxyGenerator=new RandomProxyGenerator();
    proxyGenerator.addProxy("127.0.0.1",8080,Proxy.Type.SOCKS);
    requester.setProxyGenerator(proxyGenerator);
    */

    /*??*/
    //        crawler.setResumable(true);
    crawler.setResumable(false);

    crawler.start(2);
}

From source file:icevaluation.BingAPIAccess.java

public static void main(String[] args) {
    String searchText = "arts site:wikipedia.org";
    searchText = searchText.replaceAll(" ", "%20");
    // String accountKey="jTRIJt9d8DR2QT/Z3BJCAvY1BfoXj0zRYgSZ8deqHHo";
    String accountKey = "JfeJSA3x6CtsyVai0+KEP0A6CYEUBT8VWhZmm9CS738";

    byte[] accountKeyBytes = Base64.encodeBase64((accountKey + ":" + accountKey).getBytes());
    String accountKeyEnc = new String(accountKeyBytes);
    URL url;
    try {/*from   w ww .  j av  a  2 s  .c o m*/
        url = new URL("https://api.datamarket.azure.com/Bing/Search/v1/Composite?Sources=%27Web%27&Query=%27"
                + searchText + "%27&$format=JSON");
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setRequestMethod("GET");
        conn.setRequestProperty("Authorization", "Basic " + accountKeyEnc);

        BufferedReader br = new BufferedReader(new InputStreamReader((conn.getInputStream())));
        StringBuilder sb = new StringBuilder();
        String output;
        System.out.println("Output from Server .... \n");
        //write json to string sb
        int c = 0;
        if ((output = br.readLine()) != null) {
            System.out.println("Output is: " + output);
            sb.append(output);
            c++;
            //System.out.println("C:"+c);

        }

        conn.disconnect();
        //find webtotal among output      
        int find = sb.indexOf("\"WebTotal\":\"");
        int startindex = find + 12;
        System.out.println("Find: " + find);

        int lastindex = sb.indexOf("\",\"WebOffset\"");

        System.out.println(sb.substring(startindex, lastindex));

    } catch (MalformedURLException e1) {
        e1.printStackTrace();
    } catch (IOException e) {

        e.printStackTrace();
    }

}

From source file:com.doculibre.constellio.lang.html.HtmlLangDetector.java

public static void main(String[] args) throws Exception {
    //java.net.URL url = new java.net.URL("http://www.gouv.qc.ca");
    java.net.URL url = new java.net.URL("http://andrew.triumf.ca/multilingual/samples/french.meta.html");
    java.net.URLConnection connection = url.openConnection();
    java.io.InputStream is = null;
    String content = null;//  ww  w . j  av a  2s  . c o  m
    try {
        is = connection.getInputStream();
        content = IOUtils.toString(is);
    } finally {
        IOUtils.closeQuietly(is);
    }

    HtmlLangDetector langDetector = new HtmlLangDetector();
    System.out.println(langDetector.getLang(content));
}