List of usage examples for java.lang String String
public String(StringBuilder builder)
From source file:AmazonKinesisGet.java
public static void main(String[] args) throws Exception { init();//from ww w.j av a 2 s. com final String myStreamName = "philsteststream"; final Integer myStreamSize = 1; // list all of my streams ListStreamsRequest listStreamsRequest = new ListStreamsRequest(); listStreamsRequest.setLimit(10); ListStreamsResult listStreamsResult = kinesisClient.listStreams(listStreamsRequest); List<String> streamNames = listStreamsResult.getStreamNames(); while (listStreamsResult.isHasMoreStreams()) { if (streamNames.size() > 0) { listStreamsRequest.setExclusiveStartStreamName(streamNames.get(streamNames.size() - 1)); } listStreamsResult = kinesisClient.listStreams(listStreamsRequest); streamNames.addAll(listStreamsResult.getStreamNames()); } LOG.info("Printing my list of streams : "); // print all of my streams. if (!streamNames.isEmpty()) { System.out.println("List of my streams: "); } for (int i = 0; i < streamNames.size(); i++) { System.out.println(streamNames.get(i)); } //System.out.println(streamNames.get(0)); String myownstream = streamNames.get(0); // Retrieve the Shards from a Stream DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); describeStreamRequest.setStreamName(myownstream); DescribeStreamResult describeStreamResult; List<Shard> shards = new ArrayList<>(); String lastShardId = null; do { describeStreamRequest.setExclusiveStartShardId(lastShardId); describeStreamResult = kinesisClient.describeStream(describeStreamRequest); shards.addAll(describeStreamResult.getStreamDescription().getShards()); if (shards.size() > 0) { lastShardId = shards.get(shards.size() - 1).getShardId(); } } while (describeStreamResult.getStreamDescription().getHasMoreShards()); // Get Data from the Shards in a Stream // Hard-coded to use only 1 shard String shardIterator; GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); getShardIteratorRequest.setStreamName(myownstream); //get(0) shows hardcoded to 1 stream getShardIteratorRequest.setShardId(shards.get(0).getShardId()); // using TRIM_HORIZON but could use alternatives getShardIteratorRequest.setShardIteratorType("TRIM_HORIZON"); GetShardIteratorResult getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest); shardIterator = getShardIteratorResult.getShardIterator(); // Continuously read data records from shard. List<Record> records; while (true) { // Create new GetRecordsRequest with existing shardIterator. // Set maximum records to return to 1000. GetRecordsRequest getRecordsRequest = new GetRecordsRequest(); getRecordsRequest.setShardIterator(shardIterator); getRecordsRequest.setLimit(1000); GetRecordsResult result = kinesisClient.getRecords(getRecordsRequest); // Put result into record list. Result may be empty. records = result.getRecords(); // Print records for (Record record : records) { ByteBuffer byteBuffer = record.getData(); System.out.println(String.format("Seq No: %s - %s", record.getSequenceNumber(), new String(byteBuffer.array()))); } try { Thread.sleep(1000); } catch (InterruptedException exception) { throw new RuntimeException(exception); } shardIterator = result.getNextShardIterator(); } }
From source file:hh.learnj.test.license.test.rsacoder.RSACoder.java
/** * @param args/* w w w.j a v a 2s .co m*/ * @throws Exception */ public static void main(String[] args) throws Exception { // ? // ? Map<String, Object> keyMap = RSACoder.initKey(); // byte[] publicKey = RSACoder.getPublicKey(keyMap); // ? byte[] privateKey = RSACoder.getPrivateKey(keyMap); System.out.println("/n" + Base64.encodeBase64String(publicKey)); System.out.println("?/n" + Base64.encodeBase64String(privateKey)); System.out.println( "================,?============="); String str = "RSA??"; System.out.println("/n===========????=============="); System.out.println(":" + str); // ? byte[] code1 = RSACoder.encryptByPrivateKey(str.getBytes(), privateKey); System.out.println("??" + Base64.encodeBase64String(code1)); System.out.println("===========???=============="); // ? byte[] decode1 = RSACoder.decryptByPublicKey(code1, publicKey); System.out.println("??" + new String(decode1) + "/n/n"); System.out.println("===========????????==============/n/n"); str = "????RSA"; System.out.println(":" + str); // ? byte[] code2 = RSACoder.encryptByPublicKey(str.getBytes(), publicKey); System.out.println("===========?=============="); System.out.println("??" + Base64.encodeBase64String(code2)); System.out.println("=============??======================"); System.out.println("===========??=============="); // ?? byte[] decode2 = RSACoder.decryptByPrivateKey(code2, privateKey); System.out.println("??" + new String(decode2)); }
From source file:com.vmware.photon.controller.common.auth.AuthOIDCRegistrar.java
public static int main(String[] args) { Options options = new Options(); options.addOption(USERNAME_ARG, true, "Lightwave user name"); options.addOption(PASSWORD_ARG, true, "Password"); options.addOption(TARGET_ARG, true, "Registration Hostname or IPAddress"); // Possible // load-balancer // address options.addOption(MANAGEMENT_UI_REG_FILE_ARG, true, "Management UI Registration Path"); options.addOption(SWAGGER_UI_REG_FILE_ARG, true, "Swagger UI Registration Path"); options.addOption(HELP_ARG, false, "Help"); try {/*w w w. j a v a 2s .c o m*/ String username = null; String password = null; String registrationAddress = null; String mgmtUiRegPath = null; String swaggerUiRegPath = null; CommandLineParser parser = new DefaultParser(); CommandLine cmd = null; cmd = parser.parse(options, args); if (cmd.hasOption(HELP_ARG)) { showUsage(options); return 0; } if (cmd.hasOption(USERNAME_ARG)) { username = cmd.getOptionValue(USERNAME_ARG); } if (cmd.hasOption(PASSWORD_ARG)) { password = cmd.getOptionValue(PASSWORD_ARG); } if (cmd.hasOption(TARGET_ARG)) { registrationAddress = cmd.getOptionValue(TARGET_ARG); } if (cmd.hasOption(MANAGEMENT_UI_REG_FILE_ARG)) { mgmtUiRegPath = cmd.getOptionValue(MANAGEMENT_UI_REG_FILE_ARG); } if (cmd.hasOption(SWAGGER_UI_REG_FILE_ARG)) { swaggerUiRegPath = cmd.getOptionValue(SWAGGER_UI_REG_FILE_ARG); } if (username == null || username.trim().isEmpty()) { throw new UsageException("Error: username is not specified"); } if (password == null) { char[] passwd = System.console().readPassword("Password:"); password = new String(passwd); } DomainInfo domainInfo = DomainInfo.build(); AuthOIDCRegistrar registrar = new AuthOIDCRegistrar(domainInfo); registrar.register(registrationAddress, username, password, mgmtUiRegPath, swaggerUiRegPath); return 0; } catch (ParseException e) { System.err.println(e.getMessage()); return ERROR_PARSE_EXCEPTION; } catch (UsageException e) { System.err.println(e.getMessage()); showUsage(options); return ERROR_USAGE_EXCEPTION; } catch (AuthException e) { System.err.println(e.getMessage()); return ERROR_AUTH_EXCEPTION; } }
From source file:edu.xiyou.fruits.WebCrawler.fetcher.Fetcher.java
public static void main(String[] args) { Scheduler scheduler1 = new BloomScheduler(); scheduler1.putTasks(Arrays.asList("http://www.importnew.com/all-posts", "http://blog.csdn.net/")); Fetcher fetcher = new Fetcher(scheduler1, new Handler() { @Override//from w ww. ja v a 2s. c om public void onSuccess(Response response) { // System.out.println(((Html) response).getUrl() + response.getStatusLine()); // System.out.println(new String(((Html)response).getContent())); String path = "/home/andrew/Data/"; String fileName = path + System.currentTimeMillis(); try { FileUtils.write2File(new File(fileName), ((Html) response).getContent()); } catch (IOException e) { e.printStackTrace(); } } @Override public void onFail(Response response) { } @Override public List<String> handleAndGetLinks(Response response) { LinksList list = new LinksList(); RegexRule regexRule = new RegexRule(); regexRule.addPositive(Arrays.asList("http://blog.csdn.net/\\w+/article/details/\\d+", "http://www.importnew.com/\\d+.html", "http://blog.csdn.net/?&page=\\d+")); list.getLinkByA(Jsoup.parse(new String(response.getContent())), regexRule); return list; } }, 0); // try { // System.out.println(scheduler1.takeTasks()); // } catch (InterruptedException e) { // logger.error(e.getMessage() + "bbbbbbbbbbbbbb"); // } fetcher.fetch(); }
From source file:erigo.filepump.FilePump.java
public static void main(String[] argsI) { boolean local_bShowGUI = true; String initial_outputFolder = "."; double initial_filesPerSec = 1.0; int initial_totNumFiles = 1000; String initial_mode_str = "local"; String initial_ftpHost;//from w ww . j ava 2 s . c o m String initial_ftpUser; String initial_ftpPassword; // // Parse command line arguments // // We use the Apche Commons CLI library to handle command line // arguments. See https://commons.apache.org/proper/commons-cli/usage.html // for examples, although note that we use the more up-to-date form // (Option.builder) to create Option objects. // // 1. Setup command line options // Options options = new Options(); // Example of a Boolean option (i.e., only the flag, no argument goes with it) options.addOption("h", "help", false, "Print this message."); // The following example is for: -outputfolder <folder> Location of output files Option outputFolderOption = Option.builder("outputfolder").argName("folder").hasArg() .desc("Location of output files; this folder must exist (it will not be created); default = \"" + initial_outputFolder + "\".") .build(); options.addOption(outputFolderOption); Option filesPerSecOption = Option.builder("fps").argName("filespersec").hasArg() .desc("Desired file rate, files/sec; default = " + initial_filesPerSec + ".").build(); options.addOption(filesPerSecOption); Option totNumFilesOption = Option.builder("totnum").argName("num").hasArg().desc( "Total number of output files; use -1 for unlimited number; default = " + initial_totNumFiles + ".") .build(); options.addOption(totNumFilesOption); Option outputModeOption = Option.builder("mode").argName("mode").hasArg() .desc("Specifies output interface, one of <local|ftp|sftp>; default = " + initial_mode_str + ".") .build(); options.addOption(outputModeOption); Option ftpHostOption = Option.builder("ftphost").argName("host").hasArg() .desc("Host name, for FTP or SFTP.").build(); options.addOption(ftpHostOption); Option ftpUsernameOption = Option.builder("ftpuser").argName("user").hasArg() .desc("Username, for FTP or SFTP.").build(); options.addOption(ftpUsernameOption); Option ftpPasswordOption = Option.builder("ftppass").argName("password").hasArg() .desc("Password, for FTP or SFTP.").build(); options.addOption(ftpPasswordOption); Option autoRunOption = new Option("x", "Automatically run at startup."); options.addOption(autoRunOption); // // 2. Parse command line options // CommandLineParser parser = new DefaultParser(); CommandLine line = null; try { line = parser.parse(options, argsI); } catch (ParseException exp) { // oops, something went wrong System.err.println("Command line argument parsing failed: " + exp.getMessage()); return; } // // 3. Retrieve the command line values // if (line.hasOption("help")) { // Display help message and quit HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("FilePump", options); return; } if (line.hasOption("x")) { local_bShowGUI = false; } // Where to write the files to initial_outputFolder = line.getOptionValue("outputfolder", initial_outputFolder); // How many files per second the pump should output try { initial_filesPerSec = Double.parseDouble(line.getOptionValue("fps", "" + initial_filesPerSec)); } catch (NumberFormatException nfe) { System.err.println("\nError parsing \"fps\" (it should be a floating point value):\n" + nfe); return; } // Total number of files to write out; -1 indicates unlimited try { initial_totNumFiles = Integer.parseInt(line.getOptionValue("totnum", "" + initial_totNumFiles)); } catch (NumberFormatException nfe) { System.err.println("\nError parsing \"totnum\" (it should be an integer):\n" + nfe); return; } // Specifies how files will be written out initial_mode_str = line.getOptionValue("mode", initial_mode_str); if (!initial_mode_str.equals("local") && !initial_mode_str.equals("ftp") && !initial_mode_str.equals("sftp")) { System.err.println(new String("\nUnrecognized mode, \"" + initial_mode_str + "\"")); return; } // FTP hostname initial_ftpHost = line.getOptionValue("ftphost", ""); // FTP username initial_ftpUser = line.getOptionValue("ftpuser", ""); // FTP password initial_ftpPassword = line.getOptionValue("ftppass", ""); // Create the FilePump object new FilePump(local_bShowGUI, initial_outputFolder, initial_filesPerSec, initial_totNumFiles, initial_mode_str, initial_ftpHost, initial_ftpUser, initial_ftpPassword); }
From source file:mvm.rya.accumulo.pig.SparqlQueryPigEngine.java
public static void main(String[] args) { try {//w w w. j a v a2s. c o m Preconditions.checkArgument(args.length == 7, "Usage: java -cp <jar>:$PIG_LIB <class> sparqlFile hdfsSaveLocation cbinstance cbzk cbuser cbpassword rdfTablePrefix.\n " + "Sample command: java -cp java -cp cloudbase.pig-2.0.0-SNAPSHOT-shaded.jar:/usr/local/hadoop-etc/hadoop-0.20.2/hadoop-0.20.2-core.jar:/srv_old/hdfs-tmp/pig/pig-0.9.2/pig-0.9.2.jar:$HADOOP_HOME/conf mvm.rya.accumulo.pig.SparqlQueryPigEngine " + "tstSpqrl.query temp/engineTest stratus stratus13:2181 root password l_"); String sparql = new String(ByteStreams.toByteArray(new FileInputStream(args[0]))); String hdfsSaveLocation = args[1]; SparqlToPigTransformVisitor visitor = new SparqlToPigTransformVisitor(); visitor.setTablePrefix(args[6]); visitor.setInstance(args[2]); visitor.setZk(args[3]); visitor.setUser(args[4]); visitor.setPassword(args[5]); SparqlQueryPigEngine engine = new SparqlQueryPigEngine(); engine.setSparqlToPigTransformVisitor(visitor); engine.setInference(false); engine.setStats(false); engine.init(); engine.runQuery(sparql, hdfsSaveLocation); engine.destroy(); } catch (Exception e) { e.printStackTrace(); } }
From source file:com.netscape.cmstools.CMCSharedToken.java
public static void main(String args[]) throws Exception { boolean isVerificationMode = false; // developer debugging only Options options = createOptions();//ww w .j ava2s . c o m CommandLine cmd = null; try { CommandLineParser parser = new PosixParser(); cmd = parser.parse(options, args); } catch (Exception e) { printError(e.getMessage()); System.exit(1); } if (cmd.hasOption("help")) { printHelp(); System.exit(0); } boolean verbose = cmd.hasOption("v"); String databaseDir = cmd.getOptionValue("d", "."); String passphrase = cmd.getOptionValue("s"); if (passphrase == null) { printError("Missing passphrase"); System.exit(1); } if (verbose) { System.out.println("passphrase String = " + passphrase); System.out.println("passphrase UTF-8 bytes = "); System.out.println(Arrays.toString(passphrase.getBytes("UTF-8"))); } String tokenName = cmd.getOptionValue("h"); String tokenPassword = cmd.getOptionValue("p"); String issuanceProtCertFilename = cmd.getOptionValue("b"); String issuanceProtCertNick = cmd.getOptionValue("n"); String output = cmd.getOptionValue("o"); try { CryptoManager.initialize(databaseDir); CryptoManager manager = CryptoManager.getInstance(); CryptoToken token = CryptoUtil.getKeyStorageToken(tokenName); tokenName = token.getName(); manager.setThreadToken(token); Password password = new Password(tokenPassword.toCharArray()); token.login(password); X509Certificate issuanceProtCert = null; if (issuanceProtCertFilename != null) { if (verbose) System.out.println("Loading issuance protection certificate"); String encoded = new String(Files.readAllBytes(Paths.get(issuanceProtCertFilename))); byte[] issuanceProtCertData = Cert.parseCertificate(encoded); issuanceProtCert = manager.importCACertPackage(issuanceProtCertData); if (verbose) System.out.println("issuance protection certificate imported"); } else { // must have issuance protection cert nickname if file not provided if (verbose) System.out.println("Getting cert by nickname: " + issuanceProtCertNick); if (issuanceProtCertNick == null) { System.out.println( "Invallid command: either nickname or PEM file must be provided for Issuance Protection Certificate"); System.exit(1); } issuanceProtCert = getCertificate(tokenName, issuanceProtCertNick); } EncryptionAlgorithm encryptAlgorithm = EncryptionAlgorithm.AES_128_CBC_PAD; KeyWrapAlgorithm wrapAlgorithm = KeyWrapAlgorithm.RSA; if (verbose) System.out.println("Generating session key"); SymmetricKey sessionKey = CryptoUtil.generateKey(token, KeyGenAlgorithm.AES, 128, null, true); if (verbose) System.out.println("Encrypting passphrase"); byte iv[] = { 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1 }; byte[] secret_data = CryptoUtil.encryptUsingSymmetricKey(token, sessionKey, passphrase.getBytes("UTF-8"), encryptAlgorithm, new IVParameterSpec(iv)); if (verbose) System.out.println("Wrapping session key with issuance protection cert"); byte[] issuanceProtWrappedSessionKey = CryptoUtil.wrapUsingPublicKey(token, issuanceProtCert.getPublicKey(), sessionKey, wrapAlgorithm); // final_data takes this format: // SEQUENCE { // encryptedSession OCTET STRING, // encryptedPrivate OCTET STRING // } DerOutputStream tmp = new DerOutputStream(); tmp.putOctetString(issuanceProtWrappedSessionKey); tmp.putOctetString(secret_data); DerOutputStream out = new DerOutputStream(); out.write(DerValue.tag_Sequence, tmp); byte[] final_data = out.toByteArray(); String final_data_b64 = Utils.base64encode(final_data, true); if (final_data_b64 != null) { System.out.println("\nEncrypted Secret Data:"); System.out.println(final_data_b64); } else System.out.println("Failed to produce final data"); if (output != null) { System.out.println("\nStoring Base64 secret data into " + output); try (FileWriter fout = new FileWriter(output)) { fout.write(final_data_b64); } } if (isVerificationMode) { // developer use only PrivateKey wrappingKey = null; if (issuanceProtCertNick != null) wrappingKey = (org.mozilla.jss.crypto.PrivateKey) getPrivateKey(tokenName, issuanceProtCertNick); else wrappingKey = CryptoManager.getInstance().findPrivKeyByCert(issuanceProtCert); System.out.println("\nVerification begins..."); byte[] wrapped_secret_data = Utils.base64decode(final_data_b64); DerValue wrapped_val = new DerValue(wrapped_secret_data); // val.tag == DerValue.tag_Sequence DerInputStream wrapped_in = wrapped_val.data; DerValue wrapped_dSession = wrapped_in.getDerValue(); byte wrapped_session[] = wrapped_dSession.getOctetString(); System.out.println("wrapped session key retrieved"); DerValue wrapped_dPassphrase = wrapped_in.getDerValue(); byte wrapped_passphrase[] = wrapped_dPassphrase.getOctetString(); System.out.println("wrapped passphrase retrieved"); SymmetricKey ver_session = CryptoUtil.unwrap(token, SymmetricKey.AES, 128, SymmetricKey.Usage.UNWRAP, wrappingKey, wrapped_session, wrapAlgorithm); byte[] ver_passphrase = CryptoUtil.decryptUsingSymmetricKey(token, new IVParameterSpec(iv), wrapped_passphrase, ver_session, encryptAlgorithm); String ver_spassphrase = new String(ver_passphrase, "UTF-8"); CryptoUtil.obscureBytes(ver_passphrase, "random"); System.out.println("ver_passphrase String = " + ver_spassphrase); System.out.println("ver_passphrase UTF-8 bytes = "); System.out.println(Arrays.toString(ver_spassphrase.getBytes("UTF-8"))); if (ver_spassphrase.equals(passphrase)) System.out.println("Verification success!"); else System.out.println("Verification failure! ver_spassphrase=" + ver_spassphrase); } } catch (Exception e) { if (verbose) e.printStackTrace(); printError(e.getMessage()); System.exit(1); } }
From source file:FaceRatios.java
@SuppressWarnings("serial") public static void main(String[] args) { int r = FSDK.ActivateLibrary(FACE_SDK_LICENSE); if (r == FSDK.FSDKE_OK) { FSDK.Initialize();//from w w w.ja v a 2 s. c o m FSDK.SetFaceDetectionParameters(true, true, 384); Map<String, Map<String, ArrayList<Double>>> faceProperties = new HashMap<>(); for (String directory : new File(FACE_DIRECTORY).list()) { if (new File(FACE_DIRECTORY + directory).isDirectory()) { Map<String, ArrayList<Double>> properties = new HashMap<String, ArrayList<Double>>() { { for (String property : propertyNames) put(property, new ArrayList<Double>()); } }; File[] files = new File(FACE_DIRECTORY + directory).listFiles(); System.out.println("Analyzing " + directory + " with " + files.length + " files\n"); for (File file : files) { if (file.isFile()) { HImage imageHandle = new HImage(); FSDK.LoadImageFromFileW(imageHandle, file.getAbsolutePath()); FSDK.TFacePosition.ByReference facePosition = new FSDK.TFacePosition.ByReference(); if (FSDK.DetectFace(imageHandle, facePosition) == FSDK.FSDKE_OK) { FSDK_Features.ByReference facialFeatures = new FSDK_Features.ByReference(); FSDK.DetectFacialFeaturesInRegion(imageHandle, (FSDK.TFacePosition) facePosition, facialFeatures); Point[] featurePoints = new Point[FSDK.FSDK_FACIAL_FEATURE_COUNT]; for (int i = 0; i < FSDK.FSDK_FACIAL_FEATURE_COUNT; i++) { featurePoints[i] = new Point(0, 0); featurePoints[i].x = facialFeatures.features[i].x; featurePoints[i].y = facialFeatures.features[i].y; } double eyeDistance = featureDistance(featurePoints, FeatureID.LEFT_EYE, FeatureID.RIGHT_EYE); double rightEyeSize = featureDistance(featurePoints, FeatureID.RIGHT_EYE_INNER_CORNER, FeatureID.RIGHT_EYE_OUTER_CORNER); double leftEyeSize = featureDistance(featurePoints, FeatureID.LEFT_EYE_INNER_CORNER, FeatureID.LEFT_EYE_OUTER_CORNER); double averageEyeSize = (rightEyeSize + leftEyeSize) / 2; double mouthLength = featureDistance(featurePoints, FeatureID.MOUTH_RIGHT_CORNER, FeatureID.MOUTH_LEFT_CORNER); double mouthHeight = featureDistance(featurePoints, FeatureID.MOUTH_BOTTOM, FeatureID.MOUTH_TOP); double noseHeight = featureDistance(featurePoints, FeatureID.NOSE_BOTTOM, FeatureID.NOSE_BRIDGE); double chinHeight = featureDistance(featurePoints, FeatureID.CHIN_BOTTOM, FeatureID.MOUTH_BOTTOM); double chinToBridgeHeight = featureDistance(featurePoints, FeatureID.CHIN_BOTTOM, FeatureID.NOSE_BRIDGE); double faceContourLeft = (featurePoints[FeatureID.CHIN_BOTTOM.getIndex()].getY() - featurePoints[FeatureID.FACE_CONTOUR2.getIndex()].getY()) / (featurePoints[FeatureID.CHIN_BOTTOM.getIndex()].getX() - featurePoints[FeatureID.FACE_CONTOUR2.getIndex()].getX()); double faceContourRight = (featurePoints[FeatureID.CHIN_BOTTOM.getIndex()].getY() - featurePoints[FeatureID.FACE_CONTOUR12.getIndex()].getY()) / (featurePoints[FeatureID.CHIN_BOTTOM.getIndex()].getX() - featurePoints[FeatureID.FACE_CONTOUR12.getIndex()].getX()); double bridgeLeftEyeDistance = featureDistance(featurePoints, FeatureID.LEFT_EYE_INNER_CORNER, FeatureID.NOSE_BRIDGE); double bridgeRightEyeDistance = featureDistance(featurePoints, FeatureID.RIGHT_EYE_INNER_CORNER, FeatureID.NOSE_BRIDGE); properties.get("eyeSize/eyeDistance").add(averageEyeSize / eyeDistance); properties.get("eyeSizeDisparity") .add(Math.abs(leftEyeSize - rightEyeSize) / averageEyeSize); properties.get("bridgeToEyeDisparity") .add(Math.abs(bridgeLeftEyeDistance - bridgeRightEyeDistance) / ((bridgeLeftEyeDistance + bridgeRightEyeDistance) / 2)); properties.get("eyeDistance/mouthLength").add(eyeDistance / mouthLength); properties.get("eyeDistance/noseHeight").add(eyeDistance / noseHeight); properties.get("eyeSize/mouthLength").add(eyeDistance / mouthLength); properties.get("eyeSize/noseHeight").add(eyeDistance / noseHeight); properties.get("mouthLength/mouthHeight").add(mouthLength / mouthHeight); properties.get("chinHeight/noseHeight").add(chinHeight / noseHeight); properties.get("chinHeight/chinToBridgeHeight") .add(chinHeight / chinToBridgeHeight); properties.get("noseHeight/chinToBridgeHeight") .add(noseHeight / chinToBridgeHeight); properties.get("mouthHeight/chinToBridgeHeight") .add(mouthHeight / chinToBridgeHeight); properties.get("faceCountourAngle") .add(Math.toDegrees(Math.atan((faceContourLeft - faceContourRight) / (1 + faceContourLeft * faceContourRight)))); } FSDK.FreeImage(imageHandle); } } System.out.format("%32s\t%8s\t%8s\t%3s%n", "Property", "", "", "c"); System.out.println(new String(new char[76]).replace("\0", "-")); ArrayList<Entry<String, ArrayList<Double>>> propertyList = new ArrayList<>( properties.entrySet()); Collections.sort(propertyList, new Comparator<Entry<String, ArrayList<Double>>>() { @Override public int compare(Entry<String, ArrayList<Double>> arg0, Entry<String, ArrayList<Double>> arg1) { DescriptiveStatistics dStats0 = new DescriptiveStatistics(listToArray(arg0.getValue())); DescriptiveStatistics dStats1 = new DescriptiveStatistics(listToArray(arg1.getValue())); return new Double(dStats0.getStandardDeviation() / dStats0.getMean()) .compareTo(dStats1.getStandardDeviation() / dStats1.getMean()); } }); for (Entry<String, ArrayList<Double>> property : propertyList) { DescriptiveStatistics dStats = new DescriptiveStatistics(listToArray(property.getValue())); System.out.format("%32s\t%4f\t%4f\t%3s%n", property.getKey(), dStats.getMean(), dStats.getStandardDeviation(), Math.round(dStats.getStandardDeviation() / dStats.getMean() * 100) + "%"); } System.out.println("\n"); faceProperties.put(directory, properties); } } for (String propertyName : propertyNames) { DefaultBoxAndWhiskerCategoryDataset dataset = new DefaultBoxAndWhiskerCategoryDataset(); for (Entry<String, Map<String, ArrayList<Double>>> face : faceProperties.entrySet()) { dataset.add(face.getValue().get(propertyName), "Default Series", face.getKey()); } PropertyBoxWhisker plot = new PropertyBoxWhisker(propertyName, dataset); plot.pack(); plot.setVisible(true); } } }
From source file:org.hammer.santamaria.mapper.dataset.CKANDataSetInput.java
@SuppressWarnings({ "rawtypes", "unchecked" })
public static void main(String[] pArgs) throws Exception {
String id = "proportion-of-children-under-5-years-who-have-ever-breastfed-by-county-xls-2005-6";
String sId = EncodeURIComponent(id);
String url = "https://africaopendata.org/api/action";
BSONObject dataset = new BasicBSONObject();
dataset.put("datasource", "Test");
dataset.put("id", id);
LOG.info("---> id " + id + " - " + sId);
HttpClient client = new HttpClient();
client.getHttpConnectionManager().getParams().setParameter(ClientPNames.HANDLE_REDIRECTS, false);
LOG.info(// ww w . jav a2 s. c o m
"******************************************************************************************************");
LOG.info(" ");
LOG.info(url + PACKAGE_GET + sId);
LOG.info(" ");
LOG.info(
"******************************************************************************************************");
GetMethod method = new GetMethod(url + PACKAGE_GET + sId);
method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
method.getParams().setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");
method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler(3, false));
try {
int statusCode = client.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
throw new Exception("Method failed: " + method.getStatusLine());
}
byte[] responseBody = method.getResponseBody();
LOG.debug(new String(responseBody));
Document doc = Document.parse(new String(responseBody));
if (doc != null && doc.containsKey("result")) {
Document result = new Document();
LOG.info(doc.get("result").getClass().toString());
if (doc.get("result") instanceof Document) {
LOG.info("!!! Document result !!!!");
result = (Document) doc.get("result");
} else if (doc.get("result") instanceof ArrayList) {
LOG.info("!!! Document list !!!!");
result = (Document) (((ArrayList) doc.get("result")).get(0));
} else {
LOG.info("!!! NOT FOUND !!!!");
result = null;
}
LOG.info("result find!");
if (result != null) {
dataset.put("title", result.get("title"));
dataset.put("author", result.get("author"));
dataset.put("author_email", result.get("author_email"));
dataset.put("license_id", result.get("license_id"));
}
ArrayList<String> tags = new ArrayList<String>();
ArrayList<String> meta = new ArrayList<String>();
ArrayList<String> other_tags = new ArrayList<String>();
if (result.containsKey("author") && result.get("author") != null)
other_tags.add(result.get("author").toString());
if (result.containsKey("title") && result.get("title") != null)
other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("title").toString()));
if (result.containsKey("description") && result.get("description") != null)
other_tags.addAll(DSSUtils.GetKeyWordsFromText(result.get("description").toString()));
ArrayList<Document> resources = new ArrayList<Document>();
if (result != null && result.containsKey("resources")) {
resources = (ArrayList<Document>) result.get("resources");
for (Document resource : resources) {
if (resource.getString("format").toUpperCase().equals("JSON")) {
dataset.put("dataset-type", "JSON");
dataset.put("url", resource.get("url"));
dataset.put("created", resource.get("created"));
dataset.put("description", resource.get("description"));
dataset.put("revision_timestamp", resource.get("revision_timestamp"));
meta = DSSUtils.GetMetaByResource(resource.get("url").toString());
}
}
}
if (result != null && result.containsKey("tags")) {
ArrayList<Document> tagsFromCKAN = (ArrayList<Document>) result.get("tags");
for (Document tag : tagsFromCKAN) {
if (tag.containsKey("state") && tag.getString("state").toUpperCase().equals("ACTIVE")) {
tags.add(tag.getString("display_name").trim().toLowerCase());
} else if (tag.containsKey("display_name")) {
tags.add(tag.getString("display_name").trim().toLowerCase());
}
}
}
dataset.put("tags", tags);
dataset.put("meta", meta);
dataset.put("resources", resources);
dataset.put("other_tags", other_tags);
}
} catch (Exception e) {
e.printStackTrace();
LOG.error(e);
} finally {
method.releaseConnection();
}
//GetMetaByDocument("http://catalog.data.gov/api/action/package_show?id=1e68f387-5f1c-46c0-a0d1-46044ffef5bf");
}
From source file:CourserankConnector.java
public static void main(String[] args) throws Exception { /////////////////////////////////////// //Tagger init //MaxentTagger tagger = new MaxentTagger("models/english-left3words-distsim.tagger"); ////*from www .ja va 2 s .c om*/ //CLIENT INITIALIZATION ImportData importCourse = new ImportData(); HttpClient httpclient = new DefaultHttpClient(); httpclient = WebClientDevWrapper.wrapClient(httpclient); try { /* httpclient.getCredentialsProvider().setCredentials( new AuthScope(null, -1), new UsernamePasswordCredentials("eadrian", "eactresp1")); */ ////////////////////////////////////////////////// //Get Course Bulletin Departments page List<Course> courses = new ArrayList<Course>(); HttpGet httpget = new HttpGet("http://explorecourses.stanford.edu"); System.out.println("executing request" + httpget.getRequestLine()); HttpResponse response = httpclient.execute(httpget); HttpEntity entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); String bulletinpage = ""; //STORE RETURNED HTML TO BULLETINPAGE if (entity != null) { //System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { bulletinpage += line; //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(entity); /////////////////////////////////////////////////////////////////////////////// //Login to Courserank httpget = new HttpGet("https://courserank.com/stanford/main"); System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); String page = ""; if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { page += line; //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(entity); //////////////////////////////////////////////////// //POST REQUEST LOGIN HttpPost post = new HttpPost("https://www.courserank.com/stanford/main"); List<NameValuePair> pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("RT", "")); pairs.add(new BasicNameValuePair("action", "login")); pairs.add(new BasicNameValuePair("password", "trespass")); pairs.add(new BasicNameValuePair("username", "eaconte@stanford.edu")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); HttpResponse resp = httpclient.execute(post); HttpEntity ent = resp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); /////////////////////////////////////////////////// //THIS STEP MAY NOT BE NEEDED BUT GETS MAIN PROFILE PAGE HttpGet gethome = new HttpGet("https://www.courserank.com/stanford/home"); System.out.println("executing request" + gethome.getRequestLine()); HttpResponse gresp = httpclient.execute(gethome); HttpEntity gent = gresp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + gent.getContentLength()); InputStream i = gent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } ///////////////////////////////////////////////////////////////////////////////// //Parse Bulletin String results = getToken(bulletinpage, "RESULTS HEADER", "Additional Searches"); String[] depts = results.split("href"); //SPLIT FOR EACH DEPARTMENT LINK, ITERATE boolean ready = false; for (int i = 1; i < depts.length; i++) { //EXTRACT LINK, DEPARTMENT NAME AND ABBREVIATION String dept = new String(depts[i]); String abbr = getToken(dept, "(", ")"); String name = getToken(dept, ">", "("); name.trim(); //System.out.println(tagger.tagString(name)); String link = getToken(dept, "=\"", "\">"); System.out.println(name + " : " + abbr + " : " + link); System.out.println("======================================================================"); if (i <= 10 || i >= 127) //values to keep it to undergraduate courses. Excludes law, med, business, overseas continue; /*if (i<=46) continue; */ //Start at BIOHOP /*if (abbr.equals("INTNLREL")) ready = true; if (!ready) continue;*/ //Construct department course search URL //Then request page String URL = "http://explorecourses.stanford.edu/" + link + "&filter-term-Autumn=on&filter-term-Winter=on&filter-term-Spring=on"; httpget = new HttpGet(URL); //System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); //ystem.out.println("----------------------------------------"); //System.out.println(response.getStatusLine()); String rpage = ""; if (entity != null) { //System.out.println("Response content length: " + entity.getContentLength()); InputStream in = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(in)); String line; while ((line = br.readLine()) != null) { rpage += line; //System.out.println(line); } br.close(); in.close(); } EntityUtils.consume(entity); //Process results page List<Course> deptCourses = new ArrayList<Course>(); List<Course> result = processResultPage(rpage); deptCourses.addAll(result); //While there are more result pages, keep going boolean more = (!(result.size() == 0) && (result.get((result.size() - 1)).courseNumber < 299)); boolean morepages = anotherPage(rpage); while (morepages && more) { URL = nextURL(URL); httpget = new HttpGet(URL); //System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); //System.out.println("----------------------------------------"); //System.out.println(response.getStatusLine()); rpage = ""; if (entity != null) { //System.out.println("Response content length: " + entity.getContentLength()); InputStream in = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(in)); String line; while ((line = br.readLine()) != null) { rpage += line; //System.out.println(line); } br.close(); in.close(); } EntityUtils.consume(entity); morepages = anotherPage(rpage); result = processResultPage(rpage); deptCourses.addAll(result); more = (!(result.size() == 0) && (result.get((result.size() - 1)).courseNumber < 299)); /*String mores = more? "yes": "no"; String pagess = morepages?"yes":"no"; System.out.println("more: "+mores+" morepages: "+pagess); System.out.println("more");*/ } //Get course ratings for all department courses via courserank deptCourses = getRatings(httpclient, abbr, deptCourses); for (int j = 0; j < deptCourses.size(); j++) { Course c = deptCourses.get(j); System.out.println("" + c.title + " : " + c.rating); c.tags = name; c.code = c.code.trim(); c.department = name; c.deptAB = abbr; c.writeToDatabase(); //System.out.println(tagger.tagString(c.title)); } } if (!page.equals("")) return; /////////////////////////////////////////////////// //Get Course Bulletin Department courses /* httpget = new HttpGet("https://courserank.com/stanford/main"); System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); page = ""; if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { page +=line; //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(entity); //////////////////////////////////////////////////// //POST REQUEST LOGIN HttpPost post = new HttpPost("https://www.courserank.com/stanford/main"); List<NameValuePair> pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("RT", "")); pairs.add(new BasicNameValuePair("action", "login")); pairs.add(new BasicNameValuePair("password", "trespass")); pairs.add(new BasicNameValuePair("username", "eaconte@stanford.edu")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); HttpResponse resp = httpclient.execute(post); HttpEntity ent = resp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); /////////////////////////////////////////////////// //THIS STEP MAY NOT BE NEEDED BUT GETS MAIN PROFILE PAGE HttpGet gethome = new HttpGet("https://www.courserank.com/stanford/home"); System.out.println("executing request" + gethome.getRequestLine()); HttpResponse gresp = httpclient.execute(gethome); HttpEntity gent = gresp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + gent.getContentLength()); InputStream i = gent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } //////////////////////////////////////// //GETS FIRST PAGE OF RESULTS EntityUtils.consume(gent); post = new HttpPost("https://www.courserank.com/stanford/search_results"); pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("filter_term_currentYear", "on")); pairs.add(new BasicNameValuePair("query", "")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); resp = httpclient.execute(post); ent = resp.getEntity(); System.out.println("----------------------------------------"); String rpage = ""; if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { rpage += line; System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); //////////////////////////////////////////////////// //PARSE FIRST PAGE OF RESULTS //int index = rpage.indexOf("div class=\"searchItem"); String []classSplit = rpage.split("div class=\"searchItem"); for (int i=1; i<classSplit.length; i++) { String str = classSplit[i]; //ID String CID = getToken(str, "course?id=","\">"); // CODE String CODE = getToken(str,"class=\"code\">" ,":</"); //TITLE String NAME = getToken(str, "class=\"title\">","</"); //DESCRIP String DES = getToken(str, "class=\"description\">","</"); //TERM String TERM = getToken(str, "Terms:", "|"); //UNITS String UNITS = getToken(str, "Units:", "<br/>"); //WORKLOAD String WLOAD = getToken(str, "Workload:", "|"); //GER String GER = getToken(str, "GERs:", "</d"); //RATING int searchIndex = 0; float rating = 0; while (true) { int ratingIndex = str.indexOf("large_Full", searchIndex); if (ratingIndex ==-1) { int halfratingIndex = str.indexOf("large_Half", searchIndex); if (halfratingIndex == -1) break; else rating += .5; break; } searchIndex = ratingIndex+1; rating++; } String RATING = ""+rating; //GRADE String GRADE = getToken(str, "div class=\"unofficialGrade\">", "</"); if (GRADE.equals("NOT FOUND")) { GRADE = getToken(str, "div class=\"officialGrade\">", "</"); } //REVIEWS String REVIEWS = getToken(str, "class=\"ratings\">", " ratings"); System.out.println(""+CODE+" : "+NAME + " : "+CID); System.out.println("----------------------------------------"); System.out.println("Term: "+TERM+" Units: "+UNITS+ " Workload: "+WLOAD + " Grade: "+ GRADE); System.out.println("Rating: "+RATING+ " Reviews: "+REVIEWS); System.out.println("=========================================="); System.out.println(DES); System.out.println("=========================================="); } /////////////////////////////////////////////////// //GETS SECOND PAGE OF RESULTS post = new HttpPost("https://www.courserank.com/stanford/search_results"); pairs = new ArrayList<NameValuePair>(2); pairs.add(new BasicNameValuePair("filter_term_currentYear", "on")); pairs.add(new BasicNameValuePair("page", "2")); pairs.add(new BasicNameValuePair("query", "")); post.setEntity(new UrlEncodedFormEntity(pairs)); System.out.println("executing request" + post.getRequestLine()); resp = httpclient.execute(post); ent = resp.getEntity(); System.out.println("----------------------------------------"); if (ent != null) { System.out.println("Response content length: " + ent.getContentLength()); InputStream i = ent.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { //System.out.println(line); } br.close(); i.close(); } EntityUtils.consume(ent); /* httpget = new HttpGet("https://github.com/"); System.out.println("executing request" + httpget.getRequestLine()); response = httpclient.execute(httpget); entity = response.getEntity(); System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); page = ""; if (entity != null) { System.out.println("Response content length: " + entity.getContentLength()); InputStream i = entity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(i)); String line; while ((line = br.readLine()) != null) { page +=line; System.out.println(line); } br.close(); i.close(); }*/ EntityUtils.consume(entity); } finally { // When HttpClient instance is no longer needed, // shut down the connection manager to ensure // immediate deallocation of all system resources httpclient.getConnectionManager().shutdown(); } }