List of usage examples for java.net URL URL
public URL(String spec) throws MalformedURLException
From source file:fr.inria.edelweiss.kgdqp.core.FedQueryingCLI.java
@SuppressWarnings("unchecked") public static void main(String args[]) throws ParseException, EngineException { List<String> endpoints = new ArrayList<String>(); String queryPath = null;// ww w. j a v a2 s . co m int slice = -1; Options options = new Options(); Option helpOpt = new Option("h", "help", false, "print this message"); Option queryOpt = new Option("q", "query", true, "specify the sparql query file"); Option endpointOpt = new Option("e", "endpoints", true, "the list of federated sparql endpoint URLs"); Option groupingOpt = new Option("g", "grouping", true, "triple pattern optimisation"); Option slicingOpt = new Option("s", "slicing", true, "size of the slicing parameter"); Option versionOpt = new Option("v", "version", false, "print the version information and exit"); options.addOption(queryOpt); options.addOption(endpointOpt); options.addOption(helpOpt); options.addOption(versionOpt); options.addOption(groupingOpt); options.addOption(slicingOpt); String header = "Corese/KGRAM DQP command line interface"; String footer = "\nPlease report any issue to alban.gaignard@cnrs.fr"; CommandLineParser parser = new BasicParser(); CommandLine cmd = parser.parse(options, args); if (cmd.hasOption("h")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("kgdqp", header, options, footer, true); System.exit(0); } if (!cmd.hasOption("e")) { logger.info("You must specify at least the URL of one sparql endpoint !"); System.exit(0); } else { endpoints = new ArrayList<String>(Arrays.asList(cmd.getOptionValues("e"))); } if (!cmd.hasOption("q")) { logger.info("You must specify a path for a sparql query !"); System.exit(0); } else { queryPath = cmd.getOptionValue("q"); } if (cmd.hasOption("s")) { try { slice = Integer.parseInt(cmd.getOptionValue("s")); } catch (NumberFormatException ex) { logger.warn(cmd.getOptionValue("s") + " is not formatted as number for the slicing parameter"); logger.warn("Slicing disabled"); } } if (cmd.hasOption("v")) { logger.info("version 3.0.4-SNAPSHOT"); System.exit(0); } ///////////////// Graph graph = Graph.create(); QueryProcessDQP exec = QueryProcessDQP.create(graph); exec.setGroupingEnabled(cmd.hasOption("g")); if (slice > 0) { exec.setSlice(slice); } Provider sProv = ProviderImplCostMonitoring.create(); exec.set(sProv); for (String url : endpoints) { try { exec.addRemote(new URL(url), WSImplem.REST); } catch (MalformedURLException ex) { logger.error(url + " is not a well-formed URL"); System.exit(1); } } StringBuffer fileData = new StringBuffer(1000); BufferedReader reader = null; try { reader = new BufferedReader(new FileReader(queryPath)); } catch (FileNotFoundException ex) { logger.error("Query file " + queryPath + " not found !"); System.exit(1); } char[] buf = new char[1024]; int numRead = 0; try { while ((numRead = reader.read(buf)) != -1) { String readData = String.valueOf(buf, 0, numRead); fileData.append(readData); buf = new char[1024]; } reader.close(); } catch (IOException ex) { logger.error("Error while reading query file " + queryPath); System.exit(1); } String sparqlQuery = fileData.toString(); // Query q = exec.compile(sparqlQuery, null); // System.out.println(q); StopWatch sw = new StopWatch(); sw.start(); Mappings map = exec.query(sparqlQuery); int dqpSize = map.size(); System.out.println("--------"); long time = sw.getTime(); System.out.println(time + " " + dqpSize); }
From source file:SheetStructure.java
public static void main(String[] args) { HttpURLConnection connection = null; StringBuilder response = new StringBuilder(); //We are using Jackson JSON parser to serialize and deserialize the JSON. See http://wiki.fasterxml.com/JacksonHome //Feel free to use which ever library you prefer. ObjectMapper mapper = new ObjectMapper(); mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); String accessToken = "";//Insert your access token here. try {/* www .j a v a 2 s.c om*/ System.out.println("Starting HelloSmartsheet2: Betty's Bake Sale..."); //First Create a new sheet. String sheetName = "Betty's Bake Sale"; //We will be using POJOs to represent the REST request objects. We will convert these to and from JSON using Jackson JSON. //Their structure directly relates to the JSON that gets passed through the API. //Note that these POJOs are included as static inner classes to keep this to one file. Normally they would be broken out. Sheet newSheet = new Sheet(); newSheet.setName(sheetName); newSheet.setColumns(Arrays.asList(new Column("Baked Goods", "TEXT_NUMBER", null, true, null), new Column("Baker", "CONTACT_LIST", null, null, null), new Column("Price Per Item", "TEXT_NUMBER", null, null, null), new Column("Gluten Free?", "CHECKBOX", "FLAG", null, null), new Column("Status", "PICKLIST", null, null, Arrays.asList("Started", "Finished", "Delivered")))); connection = (HttpURLConnection) new URL(GET_SHEETS_URL).openConnection(); connection.addRequestProperty("Authorization", "Bearer " + accessToken); connection.addRequestProperty("Content-Type", "application/json"); connection.setDoOutput(true); mapper.writeValue(connection.getOutputStream(), newSheet); Result<Sheet> newSheetResult = mapper.readValue(connection.getInputStream(), new TypeReference<Result<Sheet>>() { }); newSheet = newSheetResult.getResult(); System.out.println("Sheet " + newSheet.getName() + " created, id: " + newSheet.getId()); //Now add a column: String columnName = "Delivery Date"; System.out.println("Adding column " + columnName + " to " + sheetName); Column newColumn = new Column(columnName, "DATE", 5); connection = (HttpURLConnection) new URL(SHEET_COLUMNS_URL.replace(ID, "" + newSheet.getId())) .openConnection(); connection.addRequestProperty("Authorization", "Bearer " + accessToken); connection.addRequestProperty("Content-Type", "application/json"); connection.setDoOutput(true); mapper.writeValue(connection.getOutputStream(), newColumn); Result<Column> newColumnResult = mapper.readValue(connection.getInputStream(), new TypeReference<Result<Column>>() { }); System.out.println( "Column " + newColumnResult.getResult().getTitle() + " added to " + newSheet.getName()); //Next, we will get the list of Columns from the API. We could figure this out based on what the server has returned in the result, but we'll just ask the API for it. System.out.println("Fetching " + newSheet.getName() + " sheet columns..."); connection = (HttpURLConnection) new URL(SHEET_COLUMNS_URL.replace(ID, "" + newSheet.getId())) .openConnection(); connection.addRequestProperty("Authorization", "Bearer " + accessToken); List<Column> allColumns = mapper.readValue(connection.getInputStream(), new TypeReference<List<Column>>() { }); System.out.println("Fetched."); //Now we will be adding rows System.out.println("Inserting rows into " + newSheet.getName()); List<Row> rows = new ArrayList<Row>(); rows.add(new Row(Arrays.asList(new Cell(allColumns.get(0).id, "Brownies"), new Cell(allColumns.get(1).id, "julieann@example.com"), new Cell(allColumns.get(2).id, "$1"), new Cell(allColumns.get(3).id, Boolean.TRUE), new Cell(allColumns.get(4).id, "Finished")))); rows.add(new Row(Arrays.asList(new Cell(allColumns.get(0).id, "Snickerdoodles"), new Cell(allColumns.get(1).id, "stevenelson@example.com"), new Cell(allColumns.get(2).id, "$1"), new Cell(allColumns.get(3).id, Boolean.FALSE), new Cell(allColumns.get(4).id, "Delivered"), new Cell(allColumns.get(5).id, "2013-09-04")))); rows.add(new Row(Arrays.asList(new Cell(allColumns.get(0).id, "Rice Krispy Treats"), new Cell(allColumns.get(1).id, "rickthames@example.com"), new Cell(allColumns.get(2).id, "$.50"), new Cell(allColumns.get(3).id, Boolean.TRUE), new Cell(allColumns.get(4).id, "Started")))); rows.add(new Row(Arrays.asList(new Cell(allColumns.get(0).id, "Muffins"), new Cell(allColumns.get(1).id, "sandrassmart@example.com"), new Cell(allColumns.get(2).id, "$1.50"), new Cell(allColumns.get(3).id, Boolean.FALSE), new Cell(allColumns.get(4).id, "Finished")))); rows.add(new Row(Arrays.asList(new Cell(allColumns.get(0).id, "Chocolate Chip Cookies"), new Cell(allColumns.get(1).id, "janedaniels@example.com"), new Cell(allColumns.get(2).id, "$1"), new Cell(allColumns.get(3).id, Boolean.FALSE), new Cell(allColumns.get(4).id, "Delivered"), new Cell(allColumns.get(5).id, "2013-09-05")))); rows.add(new Row(Arrays.asList(new Cell(allColumns.get(0).id, "Ginger Snaps"), new Cell(allColumns.get(1).id, "nedbarnes@example.com"), new Cell(allColumns.get(2).id, "$.50"), new Cell(allColumns.get(3).id, Boolean.TRUE), new Cell(allColumns.get(4).id, "Unknown", false)))); //Note that this one is strict=false. This is because "Unknown" was not one of the original options when the column was created. RowWrapper rowWrapper = new RowWrapper(); rowWrapper.setToBottom(true); rowWrapper.setRows(rows); connection = (HttpURLConnection) new URL(SHEET_ROWS_URL.replace(ID, "" + newSheet.getId())) .openConnection(); connection.addRequestProperty("Authorization", "Bearer " + accessToken); connection.addRequestProperty("Content-Type", "application/json"); connection.setDoOutput(true); mapper.writeValue(connection.getOutputStream(), rowWrapper); Result<List<Row>> newRowsResult = mapper.readValue(connection.getInputStream(), new TypeReference<Result<List<Row>>>() { }); System.out.println("Added " + newRowsResult.getResult().size() + " rows to " + newSheet.getName()); //Move a row to the top. System.out.println("Moving row 6 to the top."); RowWrapper moveToTop = new RowWrapper(); moveToTop.setToTop(true); connection = (HttpURLConnection) new URL( ROW_URL.replace(ID, "" + newRowsResult.getResult().get(5).getId())).openConnection(); connection.setRequestMethod("PUT"); connection.addRequestProperty("Authorization", "Bearer " + accessToken); connection.addRequestProperty("Content-Type", "application/json"); connection.setDoOutput(true); mapper.writeValue(connection.getOutputStream(), moveToTop); mapper.readValue(connection.getInputStream(), new TypeReference<Result<List<Row>>>() { }); System.out.println("Row 6 moved to top."); //Insert empty rows for spacing rows = new ArrayList<Row>(); rows.add(new Row(Arrays.asList(new Cell(allColumns.get(0).id, "")))); rows.add(new Row(Arrays.asList(new Cell(allColumns.get(0).id, "Delivered")))); rowWrapper = new RowWrapper(); rowWrapper.setToBottom(true); rowWrapper.setRows(rows); connection = (HttpURLConnection) new URL(SHEET_ROWS_URL.replace(ID, "" + newSheet.getId())) .openConnection(); connection.addRequestProperty("Authorization", "Bearer " + accessToken); connection.addRequestProperty("Content-Type", "application/json"); connection.setDoOutput(true); mapper.writeValue(connection.getOutputStream(), rowWrapper); Result<List<Row>> spacerRowsResult = mapper.readValue(connection.getInputStream(), new TypeReference<Result<List<Row>>>() { }); System.out.println("Added " + spacerRowsResult.getResult().size() + " rows to " + newSheet.getName()); //Move Delivered rows to be children of the last spacer row. System.out.println("Moving delivered rows to Delivered section..."); Long[] deliveredRowIds = new Long[] { newRowsResult.result.get(1).getId(), newRowsResult.result.get(4).getId() }; RowWrapper parentRowLocation = new RowWrapper(); parentRowLocation.setParentId(spacerRowsResult.getResult().get(1).getId()); for (Long deliveredId : deliveredRowIds) { System.out.println("Moving " + deliveredId + " to Delivered."); connection = (HttpURLConnection) new URL(ROW_URL.replace(ID, "" + deliveredId)).openConnection(); connection.setRequestMethod("PUT"); connection.addRequestProperty("Authorization", "Bearer " + accessToken); connection.addRequestProperty("Content-Type", "application/json"); connection.setDoOutput(true); mapper.writeValue(connection.getOutputStream(), parentRowLocation); mapper.readValue(connection.getInputStream(), new TypeReference<Result<List<Row>>>() { }); System.out.println("Row id " + deliveredId + " moved."); } System.out.println("Appending additional rows to items in progress..."); List<Row> siblingRows = new ArrayList<Row>(); siblingRows.add(new Row(Arrays.asList(new Cell(allColumns.get(0).id, "Scones"), new Cell(allColumns.get(1).id, "tomlively@example.com"), new Cell(allColumns.get(2).id, "$1.50"), new Cell(allColumns.get(3).id, Boolean.TRUE), new Cell(allColumns.get(4).id, "Finished")))); siblingRows.add(new Row(Arrays.asList(new Cell(allColumns.get(0).id, "Lemon Bars"), new Cell(allColumns.get(1).id, "rickthames@example.com"), new Cell(allColumns.get(2).id, "$1"), new Cell(allColumns.get(3).id, Boolean.FALSE), new Cell(allColumns.get(4).id, "Started")))); rowWrapper = new RowWrapper(); rowWrapper.setSiblingId(newRowsResult.getResult().get(3).getId()); rowWrapper.setRows(siblingRows); connection = (HttpURLConnection) new URL(SHEET_ROWS_URL.replace(ID, "" + newSheet.getId())) .openConnection(); connection.addRequestProperty("Authorization", "Bearer " + accessToken); connection.addRequestProperty("Content-Type", "application/json"); connection.setDoOutput(true); mapper.writeValue(connection.getOutputStream(), rowWrapper); Result<List<Row>> siblingRowsResult = mapper.readValue(connection.getInputStream(), new TypeReference<Result<List<Row>>>() { }); System.out.println("Added " + siblingRowsResult.getResult().size() + " rows to " + newSheet.getName()); System.out.println("Moving Status column to index 1..."); Column statusColumn = allColumns.get(4); Column moveColumn = new Column(); moveColumn.setIndex(1); moveColumn.setTitle(statusColumn.title); moveColumn.setSheetId(newSheet.getId()); moveColumn.setType(statusColumn.getType()); connection = (HttpURLConnection) new URL(COLUMN_URL.replace(ID, "" + statusColumn.getId())) .openConnection(); connection.addRequestProperty("Authorization", "Bearer " + accessToken); connection.addRequestProperty("Content-Type", "application/json"); connection.setDoOutput(true); connection.setRequestMethod("PUT"); mapper.writeValue(connection.getOutputStream(), moveColumn); Result<Column> movedColumnResult = mapper.readValue(connection.getInputStream(), new TypeReference<Result<Column>>() { }); System.out.println("Moved column " + movedColumnResult.getResult().getId()); System.out.println("Completed Hellosmartsheet2: Betty's Bake Sale."); } catch (IOException e) { InputStream is = ((HttpURLConnection) connection).getErrorStream(); if (is != null) { BufferedReader reader = new BufferedReader(new InputStreamReader(is)); String line; try { response = new StringBuilder(); while ((line = reader.readLine()) != null) { response.append(line); } reader.close(); Result<?> result = mapper.readValue(response.toString(), Result.class); System.err.println(result.message); } catch (IOException e1) { e1.printStackTrace(); } } e.printStackTrace(); } catch (Exception e) { System.out.println("Something broke: " + e.getMessage()); e.printStackTrace(); } }
From source file:eu.planets_project.tb.impl.services.util.DiscoveryUtils.java
/** * //w ww . j a v a 2 s .c o m * @param args * @throws MalformedURLException */ public static void main(String[] args) throws MalformedURLException { // If called from the command line, parse the argument as a WSDL URL. if (args.length == 1) { URL wsdl = new URL(args[0]); ServiceDescription sd = DiscoveryUtils.getServiceDescription(wsdl); System.out.print(sd.toXmlFormatted()); return; } // Otherwise, do a simple test: URL wsdls[] = new URL[] { new URL("http://127.0.0.1:8080/pserv-if-simple/AlwaysSaysValidService?wsdl"), new URL("http://127.0.0.1:8080/pserv-if-simple/PassThruMigrationService?wsdl"), new URL("http://127.0.0.1:8080/pserv-pa-sanselan/SanselanMigrate?wsdl") }; for (URL wsdl : wsdls) { ServiceDescription sd = DiscoveryUtils.getServiceDescription(wsdl); System.out.println(" Description: " + sd.toXmlFormatted()); } }
From source file:com.webkruscht.wmt.DownloadFiles.java
/** * @param args/*w w w. ja va 2s .c om*/ * @throws Exception */ public static void main(String[] args) throws Exception { WebmasterTools wmt; String filename; Date date = new Date(); SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss"); String today = sdf.format(date); getProperties(); Options options = getOptions(args); try { wmt = new WebmasterTools(username, password); for (SitesEntry entry : wmt.getUserSites()) { // only process verified sites if (entry.getVerified()) { // get download paths for site JSONObject data = wmt.getDownloadList(entry); if (data != null) { for (String prop : props) { String path = (String) data.get("TOP_QUERIES"); path += "&prop=" + prop; URL url = new URL(entry.getTitle().getPlainText()); if (options.getStartdate() != null) { path += "&db=" + options.getStartdate(); path += "&de=" + options.getEnddate(); filename = String.format("%s-%s-%s-%s-%s.csv", url.getHost(), options.getStartdate(), options.getEnddate(), prop, "TopQueries"); } else { filename = String.format("%s-%s-%s-%s.csv", url.getHost(), today, prop, "TopQueries"); } OutputStreamWriter out = new OutputStreamWriter( new FileOutputStream(filePath + filename), "UTF-8"); wmt.downloadData(path, out); out.close(); } String path = (String) data.get("TOP_PAGES"); URL url = new URL(entry.getTitle().getPlainText()); filename = String.format("%s-%s-%s.csv", url.getHost(), today, "TopQueries"); OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(filePath + filename), "UTF-8"); wmt.downloadData(path, out); out.close(); } } } } catch (Exception e) { e.printStackTrace(); throw e; } }
From source file:is.idega.idegaweb.egov.gumbo.bpm.violation.ViolationDataProviderRealWebservice.java
public static void main(String[] arguments) { try {//from www.j a v a 2 s. co m FSWebserviceBROTAMAL_Service locator = new FSWebserviceBROTAMAL_ServiceLocator(); FSWebserviceBROTAMAL_PortType port = locator.getFSWebserviceBROTAMALSoap12HttpPort( new URL(GumboConstants.WEB_SERVICE_URL + "FSWebserviceBROTAMALSoap12HttpPort")); StringBuilder ret = new StringBuilder(); GetVigtunarleyfiByKtElement parameters = new GetVigtunarleyfiByKtElement("5411850389"); VigtunarleyfiTypeUser res[] = port.getVigtunarleyfiByKt(parameters); int len = res.length; for (int i = 0; i < len; i++) { ret.append(res[i].getHeitiLeyfis()); if (i < (len - 1)) { ret.append(", "); } } } catch (ServiceException se) { se.printStackTrace(); } catch (MalformedURLException e) { e.printStackTrace(); } catch (RemoteException e) { e.printStackTrace(); } }
From source file:eu.sendregning.oxalis.Main.java
public static void main(String[] args) throws Exception { OptionParser optionParser = getOptionParser(); if (args.length == 0) { System.out.println(""); optionParser.printHelpOn(System.out); System.out.println(""); return;//from w w w. j a v a 2 s . c om } OptionSet optionSet; try { optionSet = optionParser.parse(args); } catch (Exception e) { printErrorMessage(e.getMessage()); return; } File xmlInvoice = xmlDocument.value(optionSet); if (!xmlInvoice.exists()) { printErrorMessage("XML document " + xmlInvoice + " does not exist"); return; } String recipientId = recipient.value(optionSet); String senderId = sender.value(optionSet); try { System.out.println(""); System.out.println(""); // bootstraps the Oxalis outbound module OxalisOutboundModule oxalisOutboundModule = new OxalisOutboundModule(); // creates a transmission request builder and enable tracing TransmissionRequestBuilder requestBuilder = oxalisOutboundModule.getTransmissionRequestBuilder(); requestBuilder.trace(trace.value(optionSet)); System.out.println("Trace mode of RequestBuilder: " + requestBuilder.isTraceEnabled()); // add receiver participant if (recipientId != null) { requestBuilder.receiver(new ParticipantId(recipientId)); } // add sender participant if (senderId != null) { requestBuilder.sender((new ParticipantId(senderId))); } if (docType != null && docType.value(optionSet) != null) { requestBuilder.documentType(PeppolDocumentTypeId.valueOf(docType.value(optionSet))); } if (profileType != null && profileType.value(optionSet) != null) { requestBuilder.processType(PeppolProcessTypeId.valueOf(profileType.value(optionSet))); } // Supplies the payload requestBuilder.payLoad(new FileInputStream(xmlInvoice)); // Overrides the destination URL if so requested if (optionSet.has(destinationUrl)) { String destinationString = destinationUrl.value(optionSet); URL destination; try { destination = new URL(destinationString); } catch (MalformedURLException e) { printErrorMessage("Invalid destination URL " + destinationString); return; } // Fetches the transmission method, which was overridden on the command line BusDoxProtocol busDoxProtocol = BusDoxProtocol.instanceFrom(transmissionMethod.value(optionSet)); if (busDoxProtocol == BusDoxProtocol.AS2) { String accessPointSystemIdentifier = destinationSystemId.value(optionSet); if (accessPointSystemIdentifier == null) { throw new IllegalStateException("Must specify AS2 system identifier if using AS2 protocol"); } requestBuilder.overrideAs2Endpoint(destination, accessPointSystemIdentifier); } else { throw new IllegalStateException("Unknown busDoxProtocol : " + busDoxProtocol); } } // Specifying the details completed, creates the transmission request TransmissionRequest transmissionRequest = requestBuilder.build(); // Fetches a transmitter ... Transmitter transmitter = oxalisOutboundModule.getTransmitter(); // ... and performs the transmission TransmissionResponse transmissionResponse = transmitter.transmit(transmissionRequest); // Write the transmission id and where the message was delivered System.out.printf("Message using messageId %s sent to %s using %s was assigned transmissionId %s\n", transmissionResponse.getStandardBusinessHeader().getMessageId().stringValue(), transmissionResponse.getURL().toExternalForm(), transmissionResponse.getProtocol().toString(), transmissionResponse.getTransmissionId()); String evidenceFileName = transmissionResponse.getTransmissionId().toString() + "-evidence.dat"; IOUtils.copy(new ByteArrayInputStream(transmissionResponse.getEvidenceBytes()), new FileOutputStream(evidenceFileName)); System.out.printf("Wrote transmission receipt to " + evidenceFileName); } catch (Exception e) { System.out.println(""); System.out.println("Message failed : " + e.getMessage()); //e.printStackTrace(); System.out.println(""); } }
From source file:edu.uci.ics.crawler4j.examples.login.LoginCrawlController.java
public static void main(String[] args) throws Exception { // if (args.length != 2) { // System.out.println("Needed parameters: "); // System.out.println("\t rootFolder (it will contain intermediate crawl data)"); // System.out.println("\t numberOfCralwers (number of concurrent threads)"); // return; // }//w ww . jav a2s . c om /* * crawlStorageFolder is a folder where intermediate crawl data is * stored. */ String crawlStorageFolder = "/tmp/test_crawler/"; /* * numberOfCrawlers shows the number of concurrent threads that should * be initiated for crawling. */ int numberOfCrawlers = 1; CrawlConfig config = new CrawlConfig(); config.setCrawlStorageFolder(crawlStorageFolder); /* * Be polite: Make sure that we don't send more than 1 request per * second (1000 milliseconds between requests). */ config.setPolitenessDelay(1000); /* * You can set the maximum crawl depth here. The default value is -1 for * unlimited depth */ config.setMaxDepthOfCrawling(0); /* * You can set the maximum number of pages to crawl. The default value * is -1 for unlimited number of pages */ config.setMaxPagesToFetch(1000); /* * Do you need to set a proxy? If so, you can use: * config.setProxyHost("proxyserver.example.com"); * config.setProxyPort(8080); * * If your proxy also needs authentication: * config.setProxyUsername(username); config.getProxyPassword(password); */ /* * This config parameter can be used to set your crawl to be resumable * (meaning that you can resume the crawl from a previously * interrupted/crashed crawl). Note: if you enable resuming feature and * want to start a fresh crawl, you need to delete the contents of * rootFolder manually. */ config.setResumableCrawling(false); config.setIncludeHttpsPages(true); HttpClient client = new DefaultHttpClient(); HttpResponse response = client.execute(new HttpGet("http://58921.com/user/login")); HttpEntity entity = response.getEntity(); String content = EntityUtils.toString(entity, HTTP.UTF_8); Document doc = Jsoup.parse(content); Elements elements = doc.getElementById("user_login_form").children(); Element tokenEle = elements.last(); String token = tokenEle.val(); System.out.println(token); LoginConfiguration somesite; try { somesite = new LoginConfiguration("58921.com", new URL("http://58921.com/user/login"), new URL("http://58921.com/user/login/ajax?ajax=submit&__q=user/login")); somesite.addParam("form_id", "user_login_form"); somesite.addParam("mail", "paxbeijing@gmail.com"); somesite.addParam("pass", "cetas123"); somesite.addParam("submit", ""); somesite.addParam("form_token", token); config.addLoginConfiguration(somesite); } catch (MalformedURLException e) { e.printStackTrace(); } /* * Instantiate the controller for this crawl. */ PageFetcher pageFetcher = new PageFetcher(config); RobotstxtConfig robotstxtConfig = new RobotstxtConfig(); robotstxtConfig.setEnabled(false); RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher); CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer); /* * For each crawl, you need to add some seed urls. These are the first * URLs that are fetched and then the crawler starts following links * which are found in these pages */ controller.addSeed("http://58921.com/alltime?page=60"); /* * Start the crawl. This is a blocking operation, meaning that your code * will reach the line after this only when crawling is finished. */ controller.start(LoginCrawler.class, numberOfCrawlers); controller.env.close(); }
From source file:com.griddynamics.jagger.JaggerLauncher.java
public static void main(String[] args) throws Exception { Thread memoryMonitorThread = new Thread("memory-monitor") { @Override/* w w w . ja va2s.c om*/ public void run() { for (;;) { try { log.info("Memory info: totalMemory={}, freeMemory={}", Runtime.getRuntime().totalMemory(), Runtime.getRuntime().freeMemory()); Thread.sleep(60000); } catch (InterruptedException e) { throw new RuntimeException(e); } } } }; memoryMonitorThread.setDaemon(true); memoryMonitorThread.start(); String pid = ManagementFactory.getRuntimeMXBean().getName(); System.out.println(String.format("PID:%s", pid)); Properties props = System.getProperties(); for (Map.Entry<Object, Object> prop : props.entrySet()) { log.info("{}: '{}'", prop.getKey(), prop.getValue()); } log.info(""); URL directory = new URL("file:" + System.getProperty("user.dir") + "/"); loadBootProperties(directory, args[0], environmentProperties); log.debug("Bootstrap properties:"); for (String propName : environmentProperties.stringPropertyNames()) { log.debug(" {}={}", propName, environmentProperties.getProperty(propName)); } String[] roles = environmentProperties.getProperty(ROLES).split(","); Set<String> rolesSet = Sets.newHashSet(roles); if (rolesSet.contains(Role.COORDINATION_SERVER.toString())) { launchCoordinationServer(directory); } if (rolesSet.contains(Role.HTTP_COORDINATION_SERVER.toString())) { launchCometdCoordinationServer(directory); } if (rolesSet.contains(Role.RDB_SERVER.toString())) { launchRdbServer(directory); } if (rolesSet.contains(Role.MASTER.toString())) { launchMaster(directory); } if (rolesSet.contains(Role.KERNEL.toString())) { launchKernel(directory); } if (rolesSet.contains(Role.REPORTER.toString())) { launchReporter(directory); } LaunchManager launchManager = builder.build(); int result = launchManager.launch(); System.exit(result); }
From source file:io.amient.kafka.metrics.DiscoveryTool.java
public static void main(String[] args) throws IOException { OptionParser parser = new OptionParser(); parser.accepts("help", "Print usage help"); OptionSpec<String> zookeeper = parser.accepts("zookeeper", "Address of the seed zookeeper server") .withRequiredArg().required(); OptionSpec<String> dashboard = parser .accepts("dashboard", "Grafana dashboard name to be used in all generated configs") .withRequiredArg().required(); OptionSpec<String> dashboardPath = parser .accepts("dashboard-path", "Grafana location, i.e. `./instance/.data/grafana/dashboards`") .withRequiredArg();//from ww w. j av a2 s . c om OptionSpec<String> topic = parser.accepts("topic", "Name of the metrics topic to consume measurements from") .withRequiredArg(); OptionSpec<String> influxdb = parser .accepts("influxdb", "InfluxDB connect URL (including user and password)").withRequiredArg(); OptionSpec<String> interval = parser.accepts("interval", "JMX scanning interval in seconds") .withRequiredArg().defaultsTo("10"); //TODO --influxdb-database (DEFAULT_DATABASE) //TODO --dashboard-datasource (DEFAULT_DATASOURCE) if (args.length == 0 || args[0] == "-h" || args[0] == "--help") { parser.printHelpOn(System.err); System.exit(0); } OptionSet opts = parser.parse(args); try { DiscoveryTool tool = new DiscoveryTool(opts.valueOf(zookeeper)); try { List<String> topics = tool.getKafkaTopics(); List<Broker> brokers = tool.getKafkaBrokers(); int interval_s = Integer.parseInt(opts.valueOf(interval)); if (opts.has(dashboard) && opts.has(dashboardPath)) { tool.generateDashboard(opts.valueOf(dashboard), brokers, topics, DEFAULT_DATASOURCE, opts.valueOf(dashboardPath), interval_s).save(); } if (opts.has(topic)) { //producer/reporter settings System.out.println("kafka.metrics.topic=" + opts.valueOf(topic)); System.out.println("kafka.metrics.polling.interval=" + interval_s + "s"); //TODO --producer-bootstrap for truly non-intrusive agent deployment, // i.e. when producing to a different cluster from the one being discovered System.out.println("kafka.metrics.bootstrap.servers=" + brokers.get(0).hostPort()); //consumer settings System.out.println("consumer.topic=" + opts.valueOf(topic)); System.out.println("consumer.zookeeper.connect=" + opts.valueOf(zookeeper)); System.out.println("consumer.group.id=kafka-metrics-" + opts.valueOf(dashboard)); } if (!opts.has(influxdb) || !opts.has(topic)) { tool.generateScannerConfig(brokers, opts.valueOf(dashboard), interval_s).list(System.out); } if (opts.has(influxdb)) { URL url = new URL(opts.valueOf(influxdb)); System.out.println("influxdb.database=" + DEFAULT_DATABASE); System.out.println("influxdb.url=" + url.toString()); if (url.getUserInfo() != null) { System.out.println("influxdb.username=" + url.getUserInfo().split(":")[0]); if (url.getUserInfo().contains(":")) { System.out.println("influxdb.password=" + url.getUserInfo().split(":")[1]); } } } System.out.flush(); } catch (IOException e) { e.printStackTrace(); System.exit(3); } finally { tool.close(); } } catch (Exception e) { e.printStackTrace(); System.exit(2); } }
From source file:com.fusesource.customer.wssec.client.Main.java
public static void main(String args[]) throws Exception { try {// ww w .j a v a 2 s . c o m CommandLine cli = new PosixParser().parse(opts, args); timestamp = cli.hasOption("timestamp"); encrypt = cli.hasOption("encrypt"); sign = cli.hasOption("sign"); usernameToken = cli.hasOption("username-token"); passwordDigest = cli.hasOption("password-digest"); user = cli.getOptionValue("user"); pw = cli.getOptionValue("pw"); disableCNCheck = !cli.hasOption("ecnc"); if (cli.hasOption("help") || !(sign | encrypt | usernameToken | timestamp)) { printUsageAndExit(); } if (sign) { sigCertAlias = cli.getOptionValue("sa"); sigCertPw = cli.getOptionValue("spw"); sigKsLoc = cli.getOptionValue("sk"); sigKsPw = cli.getOptionValue("skpw"); if (sigCertAlias == null || sigKsLoc == null || sigKsPw == null || sigCertPw == null) { printUsageAndExit( "You must provide keystore, keystore password, cert alias and cert password for signing certificate"); } } if (encrypt) { encCertAlias = cli.getOptionValue("ea"); encKsLoc = cli.getOptionValue("ek"); encKsPw = cli.getOptionValue("ekpw"); if (encCertAlias == null || encKsLoc == null || encKsPw == null) { printUsageAndExit( "You must provide keystore, keystore password, and cert alias for encryption certificate"); } } } catch (ParseException ex) { printUsageAndExit(); } // Here we set the truststore for the client - by trusting the CA (in the // truststore.jks file) we implicitly trust all services presenting certificates // signed by this CA. // System.setProperty("javax.net.ssl.trustStore", "../certs/truststore.jks"); System.setProperty("javax.net.ssl.trustStorePassword", "truststore"); URL wsdl = new URL("https://localhost:8443/cxf/Customers?wsdl"); // The demo certs provided with this example configure the server with a certificate // called 'fuse-esb'. As this probably won't match the fully-qualified domain // name of the machine you're running on, we need to disable Common Name matching // to allow the JVM runtime to happily resolve the WSDL for the server. Note that // we also have to do something similar on the CXf proxy itself (see below). // if (disableCNCheck) { HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() { public boolean verify(String string, SSLSession ssls) { return true; } }); } // Initialise the bus // Bus bus = SpringBusFactory.newInstance().createBus(); SpringBusFactory.setDefaultBus(bus); // Define the properties to configure the WS Security Handler // Map<String, Object> props = new HashMap<String, Object>(); props.put(WSHandlerConstants.ACTION, getWSSecActions()); // Specify the callback handler for passwords. // PasswordCallback passwords = new PasswordCallback(); props.put(WSHandlerConstants.PW_CALLBACK_REF, passwords); if (usernameToken) { passwords.addUser(user, pw); props.put(WSHandlerConstants.USER, user); props.put(WSHandlerConstants.PASSWORD_TYPE, passwordDigest ? "PasswordDigest" : "PasswordText"); } if (encrypt) { props.put(WSHandlerConstants.ENCRYPTION_USER, encCertAlias); props.put(WSHandlerConstants.ENC_PROP_REF_ID, "encProps"); props.put("encProps", merlinCrypto(encKsLoc, encKsPw, encCertAlias)); props.put(WSHandlerConstants.ENC_KEY_ID, "IssuerSerial"); props.put(WSHandlerConstants.ENCRYPTION_PARTS, TIMESTAMP_AND_BODY); } if (sign) { props.put(WSHandlerConstants.SIGNATURE_USER, sigCertAlias); props.put(WSHandlerConstants.SIG_PROP_REF_ID, "sigProps"); props.put("sigProps", merlinCrypto(sigKsLoc, sigKsPw, sigCertAlias)); props.put(WSHandlerConstants.SIG_KEY_ID, "DirectReference"); props.put(WSHandlerConstants.SIGNATURE_PARTS, TIMESTAMP_AND_BODY); passwords.addUser(sigCertAlias, sigCertPw); } // Here we add the WS Security interceptor to perform security processing // on the outgoing SOAP messages. Also, we configure a logging interceptor // to log the message payload for inspection. // bus.getOutInterceptors().add(new WSS4JOutInterceptor(props)); bus.getOutInterceptors().add(new LoggingOutInterceptor()); CustomerService svc = new CustomerService_Service(wsdl).getPort( new QName("http://demo.fusesource.com/wsdl/CustomerService/", "SOAPOverHTTP"), CustomerService.class); // The demo certs provided with this example configure the server with a certificate // called 'fuse-esb'. As this probably won't match the fully-qualified domain // name of the machine you're running on, we need to disable Common Name matching // to allow the CXF runtime to happily invoke on the server. // if (disableCNCheck) { HTTPConduit httpConduit = (HTTPConduit) ClientProxy.getClient(svc).getConduit(); TLSClientParameters tls = new TLSClientParameters(); tls.setDisableCNCheck(true); httpConduit.setTlsClientParameters(tls); } System.out.println("Looking up the customer..."); // Here's the part where we invoke on the web service. // Customer c = svc.lookupCustomer("007"); System.out.println("Got customer " + c.getFirstName()); }