List of usage examples for org.apache.http.client.methods HttpGet HttpGet
public HttpGet(final String uri)
From source file:forcalibre.ClientProxyAuthentication.java
public static void main(String[] args) throws Exception { CredentialsProvider credsProvider = new BasicCredentialsProvider(); credsProvider.setCredentials(new AuthScope(PROXY_HOST, PROXY_PORT), ///new UsernamePasswordCredentials("vimpelcom_main\\vyualeksandrov", "s1teRlin9g19") new NTCredentials("vyualeksandrov", "s1teRlin9g19", PROXY_HOST, "vimpelcom_main")); Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create() .register(AuthSchemes.NTLM, new NTLMSchemeFactory()) .register(AuthSchemes.BASIC, new BasicSchemeFactory()) .register(AuthSchemes.DIGEST, new DigestSchemeFactory()) .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory()) .register(AuthSchemes.KERBEROS, new KerberosSchemeFactory()).build(); CloseableHttpClient httpclient = HttpClients.custom().setDefaultAuthSchemeRegistry(authSchemeRegistry) .setDefaultCredentialsProvider(credsProvider).build(); try {// www. j a va 2s. co m // HttpHost target = new HttpHost("www.verisign.com", 443, "https"); // HttpHost proxy = new HttpHost("localhost", 8080); HttpHost target = new HttpHost(TARGET_HOST, 80, "http"); HttpHost proxy = new HttpHost(PROXY_HOST, PROXY_PORT); RequestConfig config = RequestConfig.custom().setProxy(proxy).build(); HttpGet httpget = new HttpGet("/"); httpget.setConfig(config); System.out.println("Executing request " + httpget.getRequestLine() + " to " + target + " via " + proxy); CloseableHttpResponse response = httpclient.execute(target, httpget); try { System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); EntityUtils.consume(response.getEntity()); } finally { response.close(); } } finally { httpclient.close(); } }
From source file:com.dlmu.heipacker.crawler.client.ClientGZipContentCompression.java
public final static void main(String[] args) throws Exception { DefaultHttpClient httpclient = new DefaultHttpClient(); try {/*from w w w. java 2 s.c o m*/ httpclient.addRequestInterceptor(new HttpRequestInterceptor() { public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { if (!request.containsHeader("Accept-Encoding")) { request.addHeader("Accept-Encoding", "gzip"); } } }); httpclient.addResponseInterceptor(new HttpResponseInterceptor() { public void process(final HttpResponse response, final HttpContext context) throws HttpException, IOException { HttpEntity entity = response.getEntity(); if (entity != null) { Header ceheader = entity.getContentEncoding(); if (ceheader != null) { HeaderElement[] codecs = ceheader.getElements(); for (int i = 0; i < codecs.length; i++) { if (codecs[i].getName().equalsIgnoreCase("gzip")) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); return; } } } } } }); HttpGet httpget = new HttpGet("http://www.apache.org/"); // Execute HTTP request System.out.println("executing request " + httpget.getURI()); HttpResponse response = httpclient.execute(httpget); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); System.out.println(response.getLastHeader("Content-Encoding")); System.out.println(response.getLastHeader("Content-Length")); System.out.println("----------------------------------------"); HttpEntity entity = response.getEntity(); if (entity != null) { String content = EntityUtils.toString(entity); System.out.println(content); System.out.println("----------------------------------------"); System.out.println("Uncompressed size: " + content.length()); } } finally { // When HttpClient instance is no longer needed, // shut down the connection manager to ensure // immediate deallocation of all system resources httpclient.getConnectionManager().shutdown(); } }
From source file:net.modelbased.proasense.storage.reader.StorageReaderScrapRateTestClient.java
public static void main(String[] args) { // Get client properties from properties file // StorageReaderMongoServiceTestClient client = new StorageReaderMongoServiceTestClient(); // client.loadClientProperties(); // Hardcoded client properties (simple test client) String STORAGE_READER_SERVICE_URL = "http://192.168.84.34:8080/storage-reader"; String QUERY_SIMPLE_SENSORID = "1000692"; String QUERY_SIMPLE_STARTTIME = "1387565891068"; String QUERY_SIMPLE_ENDTIME = "1387565996633"; String QUERY_SIMPLE_PROPERTYKEY = "value"; // Default HTTP client and common properties for requests HttpClient client = new DefaultHttpClient(); StringBuilder requestUrl = null; List<NameValuePair> params = null; String queryString = null;//from ww w . j a v a 2 s . co m // Default HTTP response and common properties for responses HttpResponse response = null; ResponseHandler<String> handler = null; int status = 0; String body = null; // Default query for simple events requestUrl = new StringBuilder(STORAGE_READER_SERVICE_URL); requestUrl.append("/query/simple/default"); params = new LinkedList<NameValuePair>(); params.add(new BasicNameValuePair("sensorId", QUERY_SIMPLE_SENSORID)); params.add(new BasicNameValuePair("startTime", QUERY_SIMPLE_STARTTIME)); params.add(new BasicNameValuePair("endTime", QUERY_SIMPLE_ENDTIME)); queryString = URLEncodedUtils.format(params, "utf-8"); requestUrl.append("?"); requestUrl.append(queryString); try { HttpGet query11 = new HttpGet(requestUrl.toString()); query11.setHeader("Content-type", "application/json"); response = client.execute(query11); // Check status code status = response.getStatusLine().getStatusCode(); if (status != 200) { throw new RuntimeException("Failed! HTTP error code: " + status); } // Get body handler = new BasicResponseHandler(); body = handler.handleResponse(response); System.out.println("SIMPLE.DEFAULT: " + body); // The result is an array of simple events serialized as JSON using Apache Thrift. // The simple events can be deserialized into Java objects using Apache Thrift. ObjectMapper mapper = new ObjectMapper(); JsonNode nodeArray = mapper.readTree(body); for (JsonNode node : nodeArray) { byte[] bytes = node.toString().getBytes(); TDeserializer deserializer = new TDeserializer(new TJSONProtocol.Factory()); SimpleEvent event = new SimpleEvent(); deserializer.deserialize(event, bytes); System.out.println(event.toString()); } } catch (Exception e) { System.out.println(e.getClass().getName() + ": " + e.getMessage()); } // Average query for simple events requestUrl = new StringBuilder(STORAGE_READER_SERVICE_URL); requestUrl.append("/query/simple/average"); params = new LinkedList<NameValuePair>(); params.add(new BasicNameValuePair("sensorId", QUERY_SIMPLE_SENSORID)); params.add(new BasicNameValuePair("startTime", QUERY_SIMPLE_STARTTIME)); params.add(new BasicNameValuePair("endTime", QUERY_SIMPLE_ENDTIME)); params.add(new BasicNameValuePair("propertyKey", QUERY_SIMPLE_PROPERTYKEY)); queryString = URLEncodedUtils.format(params, "utf-8"); requestUrl.append("?"); requestUrl.append(queryString); try { HttpGet query12 = new HttpGet(requestUrl.toString()); query12.setHeader("Content-type", "application/json"); response = client.execute(query12); // Get status code status = response.getStatusLine().getStatusCode(); if (status == 200) { // Get body handler = new BasicResponseHandler(); body = handler.handleResponse(response); System.out.println("SIMPLE.AVERAGE: " + body); } else System.out.println("Error code: " + status); } catch (Exception e) { System.out.println(e.getClass().getName() + ": " + e.getMessage()); } // Maximum query for simple events requestUrl = new StringBuilder(STORAGE_READER_SERVICE_URL); requestUrl.append("/query/simple/maximum"); params = new LinkedList<NameValuePair>(); params.add(new BasicNameValuePair("sensorId", QUERY_SIMPLE_SENSORID)); params.add(new BasicNameValuePair("startTime", QUERY_SIMPLE_STARTTIME)); params.add(new BasicNameValuePair("endTime", QUERY_SIMPLE_ENDTIME)); params.add(new BasicNameValuePair("propertyKey", QUERY_SIMPLE_PROPERTYKEY)); queryString = URLEncodedUtils.format(params, "utf-8"); requestUrl.append("?"); requestUrl.append(queryString); try { HttpGet query13 = new HttpGet(requestUrl.toString()); query13.setHeader("Content-type", "application/json"); response = client.execute(query13); // Get status code if (status == 200) { // Get body handler = new BasicResponseHandler(); body = handler.handleResponse(response); System.out.println("SIMPLE.MAXIMUM: " + body); } else System.out.println("Error code: " + status); } catch (Exception e) { System.out.println(e.getClass().getName() + ": " + e.getMessage()); } // Minimum query for simple events requestUrl = new StringBuilder(STORAGE_READER_SERVICE_URL); requestUrl.append("/query/simple/minimum"); params = new LinkedList<NameValuePair>(); params.add(new BasicNameValuePair("sensorId", QUERY_SIMPLE_SENSORID)); params.add(new BasicNameValuePair("startTime", QUERY_SIMPLE_STARTTIME)); params.add(new BasicNameValuePair("endTime", QUERY_SIMPLE_ENDTIME)); params.add(new BasicNameValuePair("propertyKey", QUERY_SIMPLE_PROPERTYKEY)); queryString = URLEncodedUtils.format(params, "utf-8"); requestUrl.append("?"); requestUrl.append(queryString); try { HttpGet query14 = new HttpGet(requestUrl.toString()); query14.setHeader("Content-type", "application/json"); response = client.execute(query14); // Get status code if (status == 200) { // Get body handler = new BasicResponseHandler(); body = handler.handleResponse(response); System.out.println("SIMPLE.MINIMUM: " + body); } else System.out.println("Error code: " + status); } catch (Exception e) { System.out.println(e.getClass().getName() + ": " + e.getMessage()); } }
From source file:org.nmdp.b12s.mac.client.http.X509Config.java
public static void main(String[] args) throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException, KeyManagementException, UnrecoverableKeyException { URL trustKeyStoreUrl = X509Config.class.getResource("/trusted.jks"); URL clientKeyStoreUri = X509Config.class.getResource("/test-client.jks"); SSLContext sslContext = SSLContexts.custom() // Configure trusted certs .loadTrustMaterial(trustKeyStoreUrl, "changeit".toCharArray()) // Configure client certificate .loadKeyMaterial(clientKeyStoreUri, "changeit".toCharArray(), "changeit".toCharArray()).build(); try (TextHttpClient httpClient = new TextHttpClient("https://macbeta.b12x.org/mac/api", sslContext)) { }/* ww w . ja va 2 s .c o m*/ // Allow TLSv1 protocol only SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, new String[] { "TLSv1" }, null, SSLConnectionSocketFactory.getDefaultHostnameVerifier()); try (CloseableHttpClient httpclient = HttpClients.custom().setSSLSocketFactory(sslsf).build()) { HttpGet httpget = new HttpGet("https://macbeta.b12x.org/mac/api/codes/AA"); System.out.println("executing request " + httpget.getRequestLine()); try (CloseableHttpResponse response = httpclient.execute(httpget)) { HttpEntity entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); if (entity != null) { Charset charset = StandardCharsets.UTF_8; for (Header contentType : response.getHeaders("Content-Type")) { System.out.println("Content-Type: " + contentType); for (String part : contentType.getValue().split(";")) { if (part.startsWith("charset=")) { String charsetName = part.split("=")[1]; charset = Charset.forName(charsetName); } } } System.out.println("Response content length: " + entity.getContentLength()); String content = EntityUtils.toString(entity, charset); System.out.println(content); } EntityUtils.consume(entity); } } }
From source file:com.example.fangyichen.demomaps.IpInfoDbClient.java
public static void main(String[] args) throws IOException { if (args.length < 2 || args.length > 3) { System.out.println("Usage: org.ipinfodb.IpInfoDbClient MODE API_KEY [IP_ADDRESS]\n" + "MODE can be either 'ip-country' or 'ip-city'.\n" + "If you don't have an API_KEY yet, you can get one for free by registering at http://www.ipinfodb.com/register.php."); return;/*ww w . j a v a2s. co m*/ } String mode = args[0]; String apiKey = args[1]; String url = "http://api.ipinfodb.com/v3/" + mode + "/?format=json&key=" + apiKey; if (args.length > 2) { String ip = args[2]; url += "&ip=" + ip; } try { HttpGet request = new HttpGet(url); HttpResponse response = HTTP_CLIENT.execute(request, new BasicHttpContext()); if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { throw new RuntimeException("IpInfoDb response is " + response.getStatusLine()); } String responseBody = EntityUtils.toString(response.getEntity()); IpCityResponse ipCityResponse = MAPPER.readValue(responseBody, IpCityResponse.class); if ("OK".equals(ipCityResponse.getStatusCode())) { System.out.println(ipCityResponse.getCountryCode() + ", " + ipCityResponse.getRegionName() + ", " + ipCityResponse.getCityName()); } else { System.out.println("API status message is '" + ipCityResponse.getStatusMessage() + "'"); } } finally { HTTP_CLIENT.getConnectionManager().shutdown(); } }
From source file:edu.uci.ics.crawler4j.examples.login.LoginCrawlController.java
public static void main(String[] args) throws Exception { // if (args.length != 2) { // System.out.println("Needed parameters: "); // System.out.println("\t rootFolder (it will contain intermediate crawl data)"); // System.out.println("\t numberOfCralwers (number of concurrent threads)"); // return; // }/*w ww .ja v a 2 s .co m*/ /* * crawlStorageFolder is a folder where intermediate crawl data is * stored. */ String crawlStorageFolder = "/tmp/test_crawler/"; /* * numberOfCrawlers shows the number of concurrent threads that should * be initiated for crawling. */ int numberOfCrawlers = 1; CrawlConfig config = new CrawlConfig(); config.setCrawlStorageFolder(crawlStorageFolder); /* * Be polite: Make sure that we don't send more than 1 request per * second (1000 milliseconds between requests). */ config.setPolitenessDelay(1000); /* * You can set the maximum crawl depth here. The default value is -1 for * unlimited depth */ config.setMaxDepthOfCrawling(0); /* * You can set the maximum number of pages to crawl. The default value * is -1 for unlimited number of pages */ config.setMaxPagesToFetch(1000); /* * Do you need to set a proxy? If so, you can use: * config.setProxyHost("proxyserver.example.com"); * config.setProxyPort(8080); * * If your proxy also needs authentication: * config.setProxyUsername(username); config.getProxyPassword(password); */ /* * This config parameter can be used to set your crawl to be resumable * (meaning that you can resume the crawl from a previously * interrupted/crashed crawl). Note: if you enable resuming feature and * want to start a fresh crawl, you need to delete the contents of * rootFolder manually. */ config.setResumableCrawling(false); config.setIncludeHttpsPages(true); HttpClient client = new DefaultHttpClient(); HttpResponse response = client.execute(new HttpGet("http://58921.com/user/login")); HttpEntity entity = response.getEntity(); String content = EntityUtils.toString(entity, HTTP.UTF_8); Document doc = Jsoup.parse(content); Elements elements = doc.getElementById("user_login_form").children(); Element tokenEle = elements.last(); String token = tokenEle.val(); System.out.println(token); LoginConfiguration somesite; try { somesite = new LoginConfiguration("58921.com", new URL("http://58921.com/user/login"), new URL("http://58921.com/user/login/ajax?ajax=submit&__q=user/login")); somesite.addParam("form_id", "user_login_form"); somesite.addParam("mail", "paxbeijing@gmail.com"); somesite.addParam("pass", "cetas123"); somesite.addParam("submit", ""); somesite.addParam("form_token", token); config.addLoginConfiguration(somesite); } catch (MalformedURLException e) { e.printStackTrace(); } /* * Instantiate the controller for this crawl. */ PageFetcher pageFetcher = new PageFetcher(config); RobotstxtConfig robotstxtConfig = new RobotstxtConfig(); robotstxtConfig.setEnabled(false); RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher); CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer); /* * For each crawl, you need to add some seed urls. These are the first * URLs that are fetched and then the crawler starts following links * which are found in these pages */ controller.addSeed("http://58921.com/alltime?page=60"); /* * Start the crawl. This is a blocking operation, meaning that your code * will reach the line after this only when crawling is finished. */ controller.start(LoginCrawler.class, numberOfCrawlers); controller.env.close(); }
From source file:CourserankConnector.java
public static void main(String[] args) throws Exception { /////////////////////////////////////// //Tagger init //MaxentTagger tagger = new MaxentTagger("models/english-left3words-distsim.tagger"); ///// www . ja v a 2s. co m //CLIENT INITIALIZATION ImportData importCourse = new ImportData(); HttpClient httpclient = new DefaultHttpClient(); httpclient = WebClientDevWrapper.wrapClient(httpclient); try { /* httpclient.getCredentialsProvider().setCredentials( new AuthScope(null, -1), new UsernamePasswordCredentials("eadrian", "eactresp1")); */ ////////////////////////////////////////////////// //Get Course Bulletin Departments page List<Course> courses = new ArrayList<Course>(); HttpGet httpget = new HttpGet("http://explorecourses.stanford.edu"); System.out.println("executing request" + httpget.getRequestLine()); HttpResponse response = httpclient.execute(httpget); HttpEntity entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); String bulletinpage = ""; //STORE RETURNED HTML TO BULLETINPAGE if (entity != null) { //System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { bulletinpage += line; //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(entity); /////////////////////////////////////////////////////////////////////////////// //Login to Courserank httpget = new HttpGet("https://courserank.com/stanford/main"); System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); String page = ""; if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { page += line; //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(entity); //////////////////////////////////////////////////// //POST REQUEST LOGIN HttpPost post = new HttpPost("https://www.courserank.com/stanford/main"); List<NameValuePair> pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("RT", "")); pairs.add(new BasicNameValuePair("action", "login")); pairs.add(new BasicNameValuePair("password", "trespass")); pairs.add(new BasicNameValuePair("username", "eaconte@stanford.edu")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); HttpResponse resp = httpclient.execute(post); HttpEntity ent = resp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); /////////////////////////////////////////////////// //THIS STEP MAY NOT BE NEEDED BUT GETS MAIN PROFILE PAGE HttpGet gethome = new HttpGet("https://www.courserank.com/stanford/home"); System.out.println("executing request" + gethome.getRequestLine()); HttpResponse gresp = httpclient.execute(gethome); HttpEntity gent = gresp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + gent.getContentLength()); InputStream i = gent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } ///////////////////////////////////////////////////////////////////////////////// //Parse Bulletin String results = getToken(bulletinpage, "RESULTS HEADER", "Additional Searches"); String[] depts = results.split("href"); //SPLIT FOR EACH DEPARTMENT LINK, ITERATE boolean ready = false; for (int i = 1; i < depts.length; i++) { //EXTRACT LINK, DEPARTMENT NAME AND ABBREVIATION String dept = new String(depts[i]); String abbr = getToken(dept, "(", ")"); String name = getToken(dept, ">", "("); name.trim(); //System.out.println(tagger.tagString(name)); String link = getToken(dept, "=\"", "\">"); System.out.println(name + " : " + abbr + " : " + link); System.out.println("======================================================================"); if (i <= 10 || i >= 127) //values to keep it to undergraduate courses. Excludes law, med, business, overseas continue; /*if (i<=46) continue; */ //Start at BIOHOP /*if (abbr.equals("INTNLREL")) ready = true; if (!ready) continue;*/ //Construct department course search URL //Then request page String URL = "http://explorecourses.stanford.edu/" + link + "&filter-term-Autumn=on&filter-term-Winter=on&filter-term-Spring=on"; httpget = new HttpGet(URL); //System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); //ystem.out.println("----------------------------------------"); //System.out.println(response.getStatusLine()); String rpage = ""; if (entity != null) { //System.out.println("Response content length: " + entity.getContentLength()); InputStream in = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(in)); String line; while ((line = br.readLine()) != null) { rpage += line; //System.out.println(line); } br.close(); in.close(); } EntityUtils.consume(entity); //Process results page List<Course> deptCourses = new ArrayList<Course>(); List<Course> result = processResultPage(rpage); deptCourses.addAll(result); //While there are more result pages, keep going boolean more = (!(result.size() == 0) && (result.get((result.size() - 1)).courseNumber < 299)); boolean morepages = anotherPage(rpage); while (morepages && more) { URL = nextURL(URL); httpget = new HttpGet(URL); //System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); //System.out.println("----------------------------------------"); //System.out.println(response.getStatusLine()); rpage = ""; if (entity != null) { //System.out.println("Response content length: " + entity.getContentLength()); InputStream in = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(in)); String line; while ((line = br.readLine()) != null) { rpage += line; //System.out.println(line); } br.close(); in.close(); } EntityUtils.consume(entity); morepages = anotherPage(rpage); result = processResultPage(rpage); deptCourses.addAll(result); more = (!(result.size() == 0) && (result.get((result.size() - 1)).courseNumber < 299)); /*String mores = more? "yes": "no"; String pagess = morepages?"yes":"no"; System.out.println("more: "+mores+" morepages: "+pagess); System.out.println("more");*/ } //Get course ratings for all department courses via courserank deptCourses = getRatings(httpclient, abbr, deptCourses); for (int j = 0; j < deptCourses.size(); j++) { Course c = deptCourses.get(j); System.out.println("" + c.title + " : " + c.rating); c.tags = name; c.code = c.code.trim(); c.department = name; c.deptAB = abbr; c.writeToDatabase(); //System.out.println(tagger.tagString(c.title)); } } if (!page.equals("")) return; /////////////////////////////////////////////////// //Get Course Bulletin Department courses /* httpget = new HttpGet("https://courserank.com/stanford/main"); System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); page = ""; if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { page +=line; //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(entity); //////////////////////////////////////////////////// //POST REQUEST LOGIN HttpPost post = new HttpPost("https://www.courserank.com/stanford/main"); List<NameValuePair> pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("RT", "")); pairs.add(new BasicNameValuePair("action", "login")); pairs.add(new BasicNameValuePair("password", "trespass")); pairs.add(new BasicNameValuePair("username", "eaconte@stanford.edu")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); HttpResponse resp = httpclient.execute(post); HttpEntity ent = resp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); /////////////////////////////////////////////////// //THIS STEP MAY NOT BE NEEDED BUT GETS MAIN PROFILE PAGE HttpGet gethome = new HttpGet("https://www.courserank.com/stanford/home"); System.out.println("executing request" + gethome.getRequestLine()); HttpResponse gresp = httpclient.execute(gethome); HttpEntity gent = gresp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + gent.getContentLength()); InputStream i = gent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } //////////////////////////////////////// //GETS FIRST PAGE OF RESULTS EntityUtils.consume(gent); post = new HttpPost("https://www.courserank.com/stanford/search_results"); pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("filter_term_currentYear", "on")); pairs.add(new BasicNameValuePair("query", "")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); resp = httpclient.execute(post); ent = resp.getEntity(); System.out.println("----------------------------------------"); String rpage = ""; if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { rpage += line; System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); //////////////////////////////////////////////////// //PARSE FIRST PAGE OF RESULTS //int index = rpage.indexOf("div class=\"searchItem"); String []classSplit = rpage.split("div class=\"searchItem"); for (int i=1; i<classSplit.length; i++) { String str = classSplit[i]; //ID String CID = getToken(str, "course?id=","\">"); // CODE String CODE = getToken(str,"class=\"code\">" ,":</"); //TITLE String NAME = getToken(str, "class=\"title\">","</"); //DESCRIP String DES = getToken(str, "class=\"description\">","</"); //TERM String TERM = getToken(str, "Terms:", "|"); //UNITS String UNITS = getToken(str, "Units:", "<br/>"); //WORKLOAD String WLOAD = getToken(str, "Workload:", "|"); //GER String GER = getToken(str, "GERs:", "</d"); //RATING int searchIndex = 0; float rating = 0; while (true) { int ratingIndex = str.indexOf("large_Full", searchIndex); if (ratingIndex ==-1) { int halfratingIndex = str.indexOf("large_Half", searchIndex); if (halfratingIndex == -1) break; else rating += .5; break; } searchIndex = ratingIndex+1; rating++; } String RATING = ""+rating; //GRADE String GRADE = getToken(str, "div class=\"unofficialGrade\">", "</"); if (GRADE.equals("NOT FOUND")) { GRADE = getToken(str, "div class=\"officialGrade\">", "</"); } //REVIEWS String REVIEWS = getToken(str, "class=\"ratings\">", " ratings"); System.out.println(""+CODE+" : "+NAME + " : "+CID); System.out.println("----------------------------------------"); System.out.println("Term: "+TERM+" Units: "+UNITS+ " Workload: "+WLOAD + " Grade: "+ GRADE); System.out.println("Rating: "+RATING+ " Reviews: "+REVIEWS); System.out.println("=========================================="); System.out.println(DES); System.out.println("=========================================="); } /////////////////////////////////////////////////// //GETS SECOND PAGE OF RESULTS post = new HttpPost("https://www.courserank.com/stanford/search_results"); pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("filter_term_currentYear", "on")); pairs.add(new BasicNameValuePair("page", "2")); pairs.add(new BasicNameValuePair("query", "")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); resp = httpclient.execute(post); ent = resp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); /* httpget = new HttpGet("https://github.com/"); System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); page = ""; if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { page +=line; System.out.println(line); } br.close(); i.close(); }*/ EntityUtils.consume(entity); } finally { // When HttpClient instance is no longer needed, // shut down the connection manager to ensure // immediate deallocation of all system resources httpclient.getConnectionManager().shutdown(); } }
From source file:org.ipinfodb.IpInfoDbClient.java
public static void main(String[] args) throws IOException { if (args.length < 3 || args.length > 4) { System.out.println("Usage: org.ipinfodb.IpInfoDbClient MODE API_KEY [IP_ADDRESS]\n" + "MODE can be either 'ip-country' or 'ip-city'.\n" + "If you don't have an API_KEY yet, you can get one for free by registering at http://www.ipinfodb.com/register.php."); return;//from w w w .j ava 2 s. c o m } //Code for Reading the multiple ip addresses from a given txt file. File inputLogFilePath = new File(args[1]); FileReader fr = new FileReader(inputLogFilePath); BufferedReader br = new BufferedReader(fr); String logLine; /*File outputFile=new File("C:/Users/Intelligrape/Desktop/New_Sample_29042014.txt"); // if file doesnt exists, then create it if (!outputFile.exists()) { outputFile.createNewFile(); } FileWriter fw = new FileWriter(outputFile,true); BufferedWriter bw = new BufferedWriter(fw); bw.write("S.no, IPAddress, Country, Region, City, Zipcode"); bw.newLine();*/ try { Class.forName("com.mysql.jdbc.Driver"); System.out.println("Driver found"); } catch (ClassNotFoundException e) { System.out.println("Driver not found: " + e); } String url_db = "jdbc:mysql://mysqldb.c7zitrf2gguq.us-east-1.rds.amazonaws.com/matse?user=shagun&password=shagun001"; Connection con = null; int insertCounter = 0; int count = 1; while ((logLine = br.readLine()) != null) { String mode = args[2]; String apiKey = args[3]; String url = "http://api.ipinfodb.com/v3/" + mode + "/?format=json&key=" + apiKey; String[] split = logLine.split(","); String ip = split[3]; url += "&ip=" + ip; try { HttpGet request = new HttpGet(url); HttpResponse response = HTTP_CLIENT.execute(request, new BasicHttpContext()); if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { throw new RuntimeException("IpInfoDb response is " + response.getStatusLine()); } String responseBody = EntityUtils.toString(response.getEntity()); IpCityResponse ipCityResponse = MAPPER.readValue(responseBody, IpCityResponse.class); if ("OK".equals(ipCityResponse.getStatusCode())) { //System.out.print(count+", "+ip+", "+ipCityResponse.getCountryCode() + ", " + ipCityResponse.getRegionName() + ", " + ipCityResponse.getCityName() + ", " + ipCityResponse.zipCode+ ", " + ipCityResponse.countryName + ", " + ipCityResponse.latitude + ", " + ipCityResponse.longitude + ", " + ipCityResponse.timeZone); //bw.write(count+", "+ip+", "+ipCityResponse.getCountryCode() + ", " + ipCityResponse.getRegionName() + ", " + ipCityResponse.getCityName() + ", " + ipCityResponse.zipCode + ", " + ipCityResponse.countryName + ", " + ipCityResponse.latitude + ", " + ipCityResponse.longitude + ", " + ipCityResponse.timeZone); try { con = DriverManager.getConnection(url_db); //System.out.println("Connected successfully"); Statement stmt = con.createStatement(); //System.out.println(logLine); //System.out.println(split.length); int result = stmt.executeUpdate("INSERT INTO LOGDATA1 VALUES('" + split[0] + "','" + split[1].trim() + "','" + split[2] + "','" + split[3] + "','" + split[4] + "','" + ipCityResponse.statusCode + "','" + ipCityResponse.countryCode + "','" + ipCityResponse.countryName + "','" + ipCityResponse.regionName + "','" + ipCityResponse.cityName + "','" + ipCityResponse.zipCode + "','" + ipCityResponse.latitude + "','" + ipCityResponse.longitude + "','" + ipCityResponse.timeZone + "')"); ++insertCounter; con.close(); System.out.println(insertCounter); } catch (SQLException e) { System.out.println("something wrong in the connection string: " + e); } } else { System.out.print("API status message is '" + ipCityResponse.getStatusMessage() + "'"); } } finally { //HTTP_CLIENT.getConnectionManager().shutdown(); } ++count; } HTTP_CLIENT.getConnectionManager().shutdown(); }
From source file:OpenDataExtractor.java
public static void main(String[] args) { System.out.println("\nSeleziona in che campo fare la ricerca: "); System.out.println("Elenco completo delle gare in formato (1)"); // cc16106a-1a65-4c34-af13-cc045d181452 System.out.println("Composizione delle commissioni giudicatrici (2)"); // c90f1ffb-c315-4f59-b0e3-b0f2f8709127 System.out.println("Registro delle imprese escluse dalle gare (3)"); // 2ea798cc-1f52-4fc8-a28e-f92a6f409cb8 System.out.println("Registro delle imprese invitate alle gare (4)"); // a124b6af-ae31-428a-8ac5-bb341feb3c46 System.out.println("Registro delle imprese che hanno partecipato alle gare (5)");// e58396cf-1145-4cb1-84a4-34311238a0c6 System.out.println("Anagrafica completa dei contratti per l'acquisizione di beni e servizi (6)"); // aa6a8664-5ef5-43eb-a563-910e25161798 System.out.println("Fornitori (7)"); // 253c8ac9-8335-4425-84c5-4a90be863c00 System.out.println("Autorizzazioni subappalti per i lavori (8)"); // 4f9ba542-5768-4b39-a92a-450c45eabd5d System.out.println("Aggiudicazioni delle gare per i lavori (9)"); // 34e298ad-3a99-4feb-9614-b9c4071b9d8e System.out.println("Dati cruscotto lavori per area (10)"); // 1ee3ea1b-58e3-48bf-8eeb-36ac313eeaf8 System.out.println("Dati cruscotto lavori per lotto (11)"); // cbededce-269f-48d2-8c25-2359bf246f42 int count = 0; int scelta;//w ww . j a va 2 s.co m String id_ref = ""; Scanner scanner; try { do { scanner = new Scanner(System.in); scelta = scanner.nextInt(); switch (scelta) { case 1: id_ref = "cc16106a-1a65-4c34-af13-cc045d181452&q="; break; case 2: id_ref = "c90f1ffb-c315-4f59-b0e3-b0f2f8709127&q="; break; case 3: id_ref = "2ea798cc-1f52-4fc8-a28e-f92a6f409cb8&q="; break; case 4: id_ref = "a124b6af-ae31-428a-8ac5-bb341feb3c46&q="; break; case 5: id_ref = "e58396cf-1145-4cb1-84a4-34311238a0c6&q="; break; case 6: id_ref = "aa6a8664-5ef5-43eb-a563-910e25161798&q="; break; case 7: id_ref = "253c8ac9-8335-4425-84c5-4a90be863c00&q="; break; case 8: id_ref = "4f9ba542-5768-4b39-a92a-450c45eabd5d&q="; break; case 9: id_ref = "34e298ad-3a99-4feb-9614-b9c4071b9d8e&q="; break; case 10: id_ref = "1ee3ea1b-58e3-48bf-8eeb-36ac313eeaf8&q="; break; case 11: id_ref = "cbededce-269f-48d2-8c25-2359bf246f42&q="; break; default: System.out.println("Numero non selezionabile"); System.out.println("Reinserisci"); break; } } while (scelta <= 0 || scelta > 11); scanner = new Scanner(System.in); System.out.println("Inserisci un parametro di ricerca: "); String record = scanner.nextLine(); String limit = "&limit=10000"; System.out.println("id di riferimento: " + id_ref); System.out.println("___________________________"); String requestString = "http://dati.openexpo2015.it/catalog/api/action/datastore_search?resource_id=" + id_ref + record + limit; HttpClient client = HttpClientBuilder.create().build(); HttpGet request = new HttpGet(requestString); HttpResponse response = client.execute(request); BufferedReader rd = new BufferedReader(new InputStreamReader(response.getEntity().getContent())); String result = ""; String resline = ""; while ((resline = rd.readLine()) != null) { result += resline; } if (result != null) { JSONObject jsonObject = new JSONObject(result); JSONObject resultJson = (JSONObject) jsonObject.get("result"); JSONArray resultJsonFields = (JSONArray) resultJson.get("fields"); ArrayList<TypeFields> type = new ArrayList<TypeFields>(); TypeFields type1; JSONObject temp; while (count < resultJsonFields.length()) { temp = (JSONObject) resultJsonFields.get(count); type1 = new TypeFields(temp.getString("id"), temp.getString("type")); type.add(type1); count++; } JSONArray arr = (JSONArray) resultJson.get("records"); count = 0; while (count < arr.length()) { System.out.println("Entry numero: " + count); temp = (JSONObject) arr.get(count); for (TypeFields temp2 : type) { System.out.println(temp2.nameType + ": " + temp.get(temp2.nameType)); } count++; System.out.println("--------------------------"); } } } catch (Exception e) { e.printStackTrace(); } }
From source file:com.dlmu.heipacker.crawler.client.ClientKerberosAuthentication.java
public static void main(String[] args) throws Exception { System.setProperty("java.security.auth.login.config", "login.conf"); System.setProperty("java.security.krb5.conf", "krb5.conf"); System.setProperty("sun.security.krb5.debug", "true"); System.setProperty("javax.security.auth.useSubjectCredsOnly", "false"); DefaultHttpClient httpclient = new DefaultHttpClient(); try {// w w w .ja v a 2 s . c o m httpclient.getAuthSchemes().register(AuthPolicy.SPNEGO, new SPNegoSchemeFactory()); Credentials use_jaas_creds = new Credentials() { public String getPassword() { return null; } public Principal getUserPrincipal() { return null; } }; httpclient.getCredentialsProvider().setCredentials(new AuthScope(null, -1, null), use_jaas_creds); HttpUriRequest request = new HttpGet("http://kerberoshost/"); HttpResponse response = httpclient.execute(request); HttpEntity entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); System.out.println("----------------------------------------"); if (entity != null) { System.out.println(EntityUtils.toString(entity)); } System.out.println("----------------------------------------"); // This ensures the connection gets released back to the manager EntityUtils.consume(entity); } finally { // When HttpClient instance is no longer needed, // shut down the connection manager to ensure // immediate deallocation of all system resources httpclient.getConnectionManager().shutdown(); } }