List of usage examples for java.net URL URL
public URL(String spec) throws MalformedURLException
From source file:com.pp.dcasajus.forkpoint.GcmSender.java
public static void main(String[] args) { if (args.length < 1 || args.length > 2 || args[0] == null) { System.err.println("usage: ./gradlew run -Pmsg=\"MESSAGE\" [-Pto=\"DEVICE_TOKEN\"]"); System.err.println(""); System.err.println(/*w ww . j ava2 s .co m*/ "Specify a test message to broadcast via GCM. If a device's GCM registration token is\n" + "specified, the message will only be sent to that device. Otherwise, the message \n" + "will be sent to all devices subscribed to the \"global\" topic."); System.err.println(""); System.err.println( "Example (Broadcast):\n" + "On Windows: .\\gradlew.bat run -Pmsg=\"<Your_Message>\"\n" + "On Linux/Mac: ./gradlew run -Pmsg=\"<Your_Message>\""); System.err.println(""); System.err.println("Example (Unicast):\n" + "On Windows: .\\gradlew.bat run -Pmsg=\"<Your_Message>\" -Pto=\"<Your_Token>\"\n" + "On Linux/Mac: ./gradlew run -Pmsg=\"<Your_Message>\" -Pto=\"<Your_Token>\""); System.exit(1); } try { // Prepare JSON containing the GCM message content. What to send and where to send. JSONObject jGcmData = new JSONObject(); JSONObject jData = new JSONObject(); jData.put("message", args[0].trim()); // Where to send GCM message. if (args.length > 1 && args[1] != null) { jGcmData.put("to", args[1].trim()); } else { jGcmData.put("to", "/topics/global"); } // What to send in GCM message. jGcmData.put("data", jData); // Create connection to send GCM Message request. URL url = new URL("https://android.googleapis.com/gcm/send"); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty("Authorization", "key=" + API_KEY); conn.setRequestProperty("Content-Type", "application/json"); conn.setRequestMethod("POST"); conn.setDoOutput(true); // Send GCM message content. OutputStream outputStream = conn.getOutputStream(); outputStream.write(jGcmData.toString().getBytes()); // Read GCM response. InputStream inputStream = conn.getInputStream(); String resp = IOUtils.toString(inputStream); System.out.println(resp); System.out.println("Check your device/emulator for notification or logcat for " + "confirmation of the receipt of the GCM message."); } catch (IOException e) { System.out.println("Unable to send GCM message."); System.out.println("Please ensure that API_KEY has been replaced by the server " + "API key, and that the device's registration token is correct (if specified)."); e.printStackTrace(); } catch (JSONException e) { e.printStackTrace(); } }
From source file:bluevia.examples.MODemo.java
/** * @param args//from w w w . j a va 2s .c om */ public static void main(String[] args) throws IOException { BufferedReader iReader = null; String apiDataFile = "API-AccessToken.ini"; String consumer_key; String consumer_secret; String registrationId; OAuthConsumer apiConsumer = null; HttpURLConnection request = null; URL moAPIurl = null; Logger logger = Logger.getLogger("moSMSDemo.class"); int i = 0; int rc = 0; Thread mThread = Thread.currentThread(); try { System.setProperty("debug", "1"); iReader = new BufferedReader(new FileReader(apiDataFile)); // Private data: consumer info + access token info + phone info consumer_key = iReader.readLine(); consumer_secret = iReader.readLine(); registrationId = iReader.readLine(); // Set up the oAuthConsumer while (true) { try { logger.log(Level.INFO, String.format("#%d: %s\n", ++i, "Requesting messages...")); apiConsumer = new DefaultOAuthConsumer(consumer_key, consumer_secret); apiConsumer.setMessageSigner(new HmacSha1MessageSigner()); moAPIurl = new URL("https://api.bluevia.com/services/REST/SMS/inbound/" + registrationId + "/messages?version=v1&alt=json"); request = (HttpURLConnection) moAPIurl.openConnection(); request.setRequestMethod("GET"); apiConsumer.sign(request); StringBuffer doc = new StringBuffer(); BufferedReader br = null; rc = request.getResponseCode(); if (rc == HttpURLConnection.HTTP_OK) { br = new BufferedReader(new InputStreamReader(request.getInputStream())); String line = br.readLine(); while (line != null) { doc.append(line); line = br.readLine(); } System.out.printf("Output message: %s\n", doc.toString()); try { JSONObject apiResponse1 = new JSONObject(doc.toString()); String aux = apiResponse1.getString("receivedSMS"); if (aux != null) { String szMessage; String szOrigin; String szDate; JSONObject smsPool = apiResponse1.getJSONObject("receivedSMS"); JSONArray smsInfo = smsPool.optJSONArray("receivedSMS"); if (smsInfo != null) { for (i = 0; i < smsInfo.length(); i++) { szMessage = smsInfo.getJSONObject(i).getString("message"); szOrigin = smsInfo.getJSONObject(i).getJSONObject("originAddress") .getString("phoneNumber"); szDate = smsInfo.getJSONObject(i).getString("dateTime"); System.out.printf("#%d %s\n - from %s\n - message:%s\n", i, szDate, szOrigin, szMessage); } } else { JSONObject sms = smsPool.getJSONObject("receivedSMS"); szMessage = sms.getString("message"); szOrigin = sms.getJSONObject("originAddress").getString("phoneNumber"); szDate = sms.getString("dateTime"); System.out.printf("#%d %s\n - from %s\n - message:%s\n", i, szDate, szOrigin, szMessage); } } } catch (JSONException e) { System.err.println("JSON error: " + e.getMessage()); } } else if (rc == HttpURLConnection.HTTP_NO_CONTENT) System.out.printf("No content\n"); else System.err.printf("Error: %d:%s\n", rc, request.getResponseMessage()); request.disconnect(); } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); } mThread.sleep(15000); } } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); } }
From source file:Main.java
public static void main(String[] args) throws Exception { anImage = new ImageIcon(ImageIO.read(new URL("http://www.java2s.com/style/download.png"))); new Main().initComponents(); }
From source file:edu.scripps.fl.pubchem.app.ResultDownloader.java
public static void main(String[] args) throws Exception { CommandLineHandler clh = new CommandLineHandler() { public void configureOptions(Options options) { options.addOption(OptionBuilder.withLongOpt("data_url").withType("").withValueSeparator('=') .hasArg().create()); }/*from w w w . j a va 2s. com*/ }; args = clh.handle(args); String data_url = clh.getCommandLine().getOptionValue("data_url"); ResultDownloader rd = new ResultDownloader(); if (data_url != null) rd.setDataUrl(new URL(data_url)); else rd.setDataUrl(new URL("ftp://ftp.ncbi.nlm.nih.gov/pubchem/Bioassay/CSV/")); if (args.length == 0) rd.process(); else { Integer[] list = (Integer[]) ConvertUtils.convert(args, Integer[].class); rd.process(((List<Integer>) Arrays.asList(list)).iterator()); } }
From source file:com.yahoo.athenz.example.ztoken.HttpExampleClient.java
public static void main(String[] args) throws MalformedURLException, IOException { // parse our command line to retrieve required input CommandLine cmd = parseCommandLine(args); String domainName = cmd.getOptionValue("domain"); String serviceName = cmd.getOptionValue("service"); String privateKeyPath = cmd.getOptionValue("pkey"); String keyId = cmd.getOptionValue("keyid"); String url = cmd.getOptionValue("url"); String ztsUrl = cmd.getOptionValue("ztsurl"); String providerDomain = cmd.getOptionValue("provider-domain"); String providerRole = cmd.getOptionValue("provider-role"); // we need to generate our principal credentials (ntoken). In // addition to the domain and service names, we need the // the service's private key and the key identifier - the // service with the corresponding public key must already be // registered in ZMS PrivateKey privateKey = Crypto.loadPrivateKey(new File(privateKeyPath)); ServiceIdentityProvider identityProvider = new SimpleServiceIdentityProvider(domainName, serviceName, privateKey, keyId);//from w w w.ja v a 2 s .c o m // now we need to retrieve a role token (ztoken) for accessing // the provider Athenz enabled service RoleToken roleToken = null; try (ZTSClient ztsClient = new ZTSClient(ztsUrl, domainName, serviceName, identityProvider)) { roleToken = ztsClient.getRoleToken(providerDomain, providerRole); } if (roleToken == null) { System.out.println( "Unable to retrieve role token for: " + providerRole + " in domain: " + providerDomain); System.exit(1); } URL obj = new URL(url); HttpURLConnection con = (HttpURLConnection) obj.openConnection(); // set our Athenz credentials. The ZTSClient provides the header // name that we must use for authorization token while the role // token itself provides the token string (ztoken). System.out.println("Using RoleToken: " + roleToken.getToken()); con.setRequestProperty(ZTSClient.getHeader(), roleToken.getToken()); // now process our request int responseCode = con.getResponseCode(); switch (responseCode) { case HttpURLConnection.HTTP_FORBIDDEN: System.out.println("Request was forbidden - not authorized: " + con.getResponseMessage()); break; case HttpURLConnection.HTTP_OK: System.out.println("Successful response: "); try (BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()))) { String inputLine; while ((inputLine = in.readLine()) != null) { System.out.println(inputLine); } } break; default: System.out.println("Request failed - response status code: " + responseCode); } }
From source file:com.jixstreet.temanusaha.gcm.GcmSender.java
public static void main(String[] args) { if (args.length < 1 || args.length > 2 || args[0] == null) { System.err.println("usage: ./gradlew run -Pmsg=\"MESSAGE\" [-Pto=\"DEVICE_TOKEN\"]"); System.err.println(""); System.err.println(/*from ww w. ja v a 2 s .c o m*/ "Specify a test message to broadcast via GCM. If a device's GCM registration token is\n" + "specified, the message will only be sent to that device. Otherwise, the message \n" + "will be sent to all devices subscribed to the \"global\" topic."); System.err.println(""); System.err.println( "Example (Broadcast):\n" + "On Windows: .\\gradlew.bat run -Pmsg=\"<Your_Message>\"\n" + "On Linux/Mac: ./gradlew run -Pmsg=\"<Your_Message>\""); System.err.println(""); System.err.println("Example (Unicast):\n" + "On Windows: .\\gradlew.bat run -Pmsg=\"<Your_Message>\" -Pto=\"<Your_Token>\"\n" + "On Linux/Mac: ./gradlew run -Pmsg=\"<Your_Message>\" -Pto=\"<Your_Token>\""); System.exit(1); } try { // Prepare JSON containing the GCM message content. What to send and where to send. JSONObject jGcmData = new JSONObject(); JSONObject jData = new JSONObject(); try { jData.put("message", args[0].trim()); } catch (JSONException e) { e.printStackTrace(); } // Where to send GCM message. if (args.length > 1 && args[1] != null) { try { jGcmData.put("to", args[1].trim()); } catch (JSONException e) { e.printStackTrace(); } } else { try { jGcmData.put("to", "/topics/global"); } catch (JSONException e) { e.printStackTrace(); } } // What to send in GCM message. try { JSONObject data = jGcmData.put("data", jData); } catch (JSONException e) { e.printStackTrace(); } // Create connection to send GCM Message request. URL url = new URL("https://android.googleapis.com/gcm/send"); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty("Authorization", "key=" + API_KEY); conn.setRequestProperty("Content-Type", "application/json"); conn.setRequestMethod("POST"); conn.setDoOutput(true); // Send GCM message content. OutputStream outputStream = conn.getOutputStream(); outputStream.write(jGcmData.toString().getBytes()); // Read GCM response. InputStream inputStream = conn.getInputStream(); String resp = IOUtils.toString(inputStream); System.out.println(resp); System.out.println("Check your device/emulator for notification or logcat for " + "confirmation of the receipt of the GCM message."); } catch (IOException e) { System.out.println("Unable to send GCM message."); System.out.println("Please ensure that API_KEY has been replaced by the server " + "API key, and that the device's registration token is correct (if specified)."); e.printStackTrace(); } }
From source file:at.gv.egiz.pdfas.web.client.test.SimpleTest.java
public static void main(String[] args) { try {/*from w w w .j a v a2 s.c om*/ FileInputStream fis = new FileInputStream("/home/afitzek/simple.pdf"); byte[] inputData = IOUtils.toByteArray(fis); PDFASSignParameters signParameters = new PDFASSignParameters(); signParameters.setConnector(Connector.JKS); signParameters.setPosition(null); signParameters.setProfile("SIGNATURBLOCK_SMALL_DE"); signParameters.setQRCodeContent("TEST CONTENT"); //signParameters.setKeyIdentifier("test"); PDFASSignRequest request = new PDFASSignRequest(); request.setInputData(inputData); request.setParameters(signParameters); request.setRequestID("SOME TEST ID"); //URL endpoint = new //URL("http://demo.egiz.gv.at/demoportal-pdf_as/wssign?wsdl"); //URL endpoint = new // URL("http://www.buergerkarte.at/pdf-as-extern-4/wssign?wsdl"); String baseUrl = "http://demo.egiz.gv.at/demoportal-pdf_as/services/"; //String baseUrl = "http://localhost:8080/pdf-as-web/services/"; //URL endpoint = new URL( // "http://192.168.56.10/pdf-as-web/wssign?wsdl"); URL signEndpoint = new URL(baseUrl + "wssign?wsdl"); URL verifyEndpoint = new URL(baseUrl + "wsverify?wsdl"); RemotePDFSigner signer = new RemotePDFSigner(signEndpoint, true); RemotePDFVerifier verifier = new RemotePDFVerifier(verifyEndpoint, true); PDFASSignRequest signrequest = new PDFASSignRequest(); signrequest.setInputData(inputData); signrequest.setParameters(signParameters); signParameters.setTransactionId("MYID ...."); System.out.println("Simple Request:"); PDFASSignResponse response = signer.signPDFDokument(signrequest); System.out.println("Sign Error: " + response.getError()); PDFASVerifyRequest verifyRequest = new PDFASVerifyRequest(); verifyRequest.setInputData(response.getSignedPDF()); verifyRequest.setVerificationLevel(VerificationLevel.INTEGRITY_ONLY); PDFASVerifyResponse verifyResponse = verifier.verifyPDFDokument(verifyRequest); List<PDFASVerifyResult> results = verifyResponse.getVerifyResults(); for (int i = 0; i < results.size(); i++) { PDFASVerifyResult result = results.get(i); printVerifyResult(result); } /* * System.out.println("Simple Request:"); byte[] outputFile = * signer.signPDFDokument(inputData, signParameters); * * FileOutputStream fos = new FileOutputStream( * "/home/afitzek/simple_request_signed.pdf"); * fos.write(outputFile); fos.close(); * * System.out.println("Simple Request Obj:"); PDFASSignResponse * response = signer.signPDFDokument(request); * * if (response.getSignedPDF() != null) { FileOutputStream fos2 = * new FileOutputStream( * "/home/afitzek/simple_request_obj_signed.pdf"); * fos2.write(response.getSignedPDF()); fos2.close(); } * * if(response.getError() != null) { System.out.println("ERROR: " + * response.getError()); } */ /* List<PDFASSignRequest> bulk = new ArrayList<PDFASSignRequest>(); for (int i = 0; i < 10; i++) { bulk.add(request); } PDFASBulkSignRequest bulkRequest = new PDFASBulkSignRequest(); bulkRequest.setSignRequests(bulk); for (int j = 0; j < 10; j++) { System.out.println("Bulk Request:"); PDFASBulkSignResponse responses = signer .signPDFDokument(bulkRequest); for (int i = 0; i < responses.getSignResponses().size(); i++) { PDFASSignResponse bulkresponse = responses .getSignResponses().get(i); System.out.println("ID: " + bulkresponse.getRequestID()); if (bulkresponse.getError() != null) { System.out.println("ERROR: " + bulkresponse.getError()); } else { System.out.println("OK"); } } } */ System.out.println("Done!"); } catch (Throwable e) { e.printStackTrace(); } }
From source file:eu.planets_project.ifr.core.servreg.utils.client.PlanetsCommand.java
/** * //from w ww.jav a 2s .co m * @param args */ public static void main(String[] args) { /* FIXME, point to log4j.properties instead of doing this? */ /* java.util.logging.Logger.getLogger("com.sun.xml.ws.model").setLevel(java.util.logging.Level.WARNING); java.util.logging.Logger.getAnonymousLogger().setLevel(java.util.logging.Level.WARNING); Logger sunlogger = Logger.getLogger("com.sun.xml.ws.model"); sunlogger.setLevel(Level.WARNING); java.util.logging.Logger.getLogger( com.sun.xml.ws.util.Constants.LoggingDomain).setLevel(java.util.logging.Level.WARNING); */ /* Lots of info please: */ java.util.logging.Logger.getAnonymousLogger().setLevel(java.util.logging.Level.FINEST); java.util.logging.Logger.getLogger(com.sun.xml.ws.util.Constants.LoggingDomain) .setLevel(java.util.logging.Level.FINEST); // TODO See https://jax-ws.dev.java.net/guide/Logging.html for info on more logging to set up. //System.setProperty("com.sun.xml.ws.transport.local.LocalTransportPipe.dump","true"); //System.setProperty("com.sun.xml.ws.util.pipe.StandaloneTubeAssembler.dump","true"); //System.setProperty("com.sun.xml.ws.transport.http.HttpAdapter.dump","true"); // Doing this KILLS STREAMING. Log that. //System.setProperty("com.sun.xml.ws.transport.http.client.HttpTransportPipe.dump","true"); URL wsdl; try { wsdl = new URL(args[0]); } catch (MalformedURLException e) { e.printStackTrace(); return; } PlanetsServiceExplorer pse = new PlanetsServiceExplorer(wsdl); System.out.println(".describe(): " + pse.getServiceDescription()); Service service = Service.create(wsdl, pse.getQName()); //service.addPort(portName, SOAPBinding.SOAP11HTTP_MTOM_BINDING, endpointAddress) PlanetsService ps = (PlanetsService) service.getPort(pse.getServiceClass()); // TODO The client wrapper code should enforce this stuff: SOAPBinding binding = (SOAPBinding) ((BindingProvider) ps).getBinding(); System.out.println("Logging MTOM=" + binding.isMTOMEnabled()); ((BindingProvider) ps).getRequestContext().put(JAXWSProperties.MTOM_THRESHOLOD_VALUE, 8192); ((BindingProvider) ps).getRequestContext().put(JAXWSProperties.HTTP_CLIENT_STREAMING_CHUNK_SIZE, 8192); System.out.println("Logging MTOM=" + binding.isMTOMEnabled()); binding.setMTOMEnabled(true); System.out.println("Logging MTOM=" + binding.isMTOMEnabled()); //System.out.println("Logging MTOM="+((BindingProvider)ps).getBinding().getBindingID()+" v. "+SOAPBinding.SOAP11HTTP_MTOM_BINDING); /* * The different services are invoked in different ways... */ if (pse.getQName().equals(Migrate.QNAME)) { System.out.println("Is a Migrate service. "); Migrate s = MigrateWrapper.createWrapper(wsdl); DigitalObject dobIn = new DigitalObject.Builder(Content.byReference(new File(args[1]))).build(); MigrateResult result = s.migrate(dobIn, URI.create(args[2]), URI.create(args[3]), null); System.out.println("ServiceReport: " + result.getReport()); DigitalObjectUtils.toFile(result.getDigitalObject(), new File("output")); } else if (pse.getQName().equals(Identify.QNAME)) { System.out.println("Is an Identify service. "); Identify s = new IdentifyWrapper(wsdl); DigitalObject dobIn = new DigitalObject.Builder(Content.byReference(new File(args[1]))).build(); IdentifyResult result = s.identify(dobIn, null); System.out.println("ServiceReport: " + result.getReport()); } }
From source file:com.hiperium.commons.client.soap.AthenticationDispatcherTest.java
/** * This class authenticates with the Hiperium Platform using JAXBContext and the selects user's home and * profile to access the system functions, and finally end the session. For all this lasts functions * we use SOAP messages with dispatchers and not JAXBContext. * /*from www. j av a 2 s . co m*/ * We need to recall, that for JAXBContext we could not add a handler to add header security parameters that need * to be verified in every service call in the Hiperium Platform, and for that reason, we only use JAXBContext in * the authentication process that do not need this header validation in the SOAP message header. * * @param args */ public static void main(String[] args) { try { URL authURL = new URL(AUTH_SERVICE_URL); // The NAMESPACE must be http://authentication.soap.web.hiperium.com/ and must not begin // with http://SEI.authentication.soap.web.hiperium.com. Furthermore, the service name // and service port name must end WSService and WSPort respectively in order to call the service. QName authServiceQName = new QName(AUTH_NAMESPACE, "UserAuthenticationWSService"); QName authPortQName = new QName(AUTH_NAMESPACE, "UserAuthenticationWSPort"); // The parameter class must be annotated with @XmlRootElement in order to function JAXBContext jaxbContext = JAXBContext.newInstance(UserAuthentication.class, UserAuthenticationResponse.class); Service authService = Service.create(authURL, authServiceQName); // When we use JAXBContext the dispatcher mode must be PAYLOAD and NOT MESSAGE mode Dispatch<Object> dispatch = authService.createDispatch(authPortQName, jaxbContext, Service.Mode.PAYLOAD); UserCredentialDTO credentialsDTO = new UserCredentialDTO("andres_solorzano85@hotmail.com", "andres123"); UserAuthentication authenticateRequest = new UserAuthentication(); authenticateRequest.setArg0(credentialsDTO); UserAuthenticationResponse authenticateResponse = (UserAuthenticationResponse) dispatch .invoke(authenticateRequest); String sessionId = authenticateResponse.getReturn(); LOGGER.debug("SESSION ID: " + sessionId); // Verify if session ID exists if (StringUtils.isNotBlank(sessionId)) { // Select home and profile URL selectHomeURL = new URL(HOME_SELECTION_SERVICE_URL); Service selectHomeService = Service.create(selectHomeURL, new QName(AUTH_NAMESPACE, "HomeSelectionWSService")); selectHomeService.setHandlerResolver(new ClientHandlerResolver(credentialsDTO.getEmail(), CLIENT_APPLICATION_TOKEN, AuthenticationObjectFactory._SelectHome_QNAME)); Dispatch<SOAPMessage> selectHomeDispatch = selectHomeService.createDispatch( new QName(AUTH_NAMESPACE, "HomeSelectionWSPort"), SOAPMessage.class, Service.Mode.MESSAGE); CommonsUtil.addSessionHeaderParms(selectHomeDispatch, sessionId); HomeSelectionDTO homeSelectionDTO = new HomeSelectionDTO(1L, 1L); SOAPMessage response = selectHomeDispatch.invoke(createSelectHomeSOAPMessage(homeSelectionDTO)); readSOAPMessageResponse(response); // End Session URL endSessionURL = new URL(END_SESSION_SERVICE_URL); QName endSessionServiceQName = new QName(GENERAL_NAMESPACE, "MenuWSService"); QName endSessionPortQName = new QName(GENERAL_NAMESPACE, "MenuWSPort"); Service endSessionService = Service.create(endSessionURL, endSessionServiceQName); endSessionService.setHandlerResolver(new ClientHandlerResolver(credentialsDTO.getEmail(), CLIENT_APPLICATION_TOKEN, GeneralObjectFactory._EndSession_QNAME)); Dispatch<SOAPMessage> endSessionDispatch = endSessionService.createDispatch(endSessionPortQName, SOAPMessage.class, Service.Mode.MESSAGE); CommonsUtil.addSessionHeaderParms(endSessionDispatch, sessionId); response = endSessionDispatch.invoke(createEndSessionSOAPMessage()); readSOAPMessageResponse(response); } } catch (MalformedURLException e) { LOGGER.error(e.getMessage(), e); } catch (JAXBException e) { LOGGER.error(e.getMessage(), e); } }
From source file:io.fluo.webindex.data.Copy.java
public static void main(String[] args) throws Exception { if (args.length != 3) { log.error("Usage: Copy <pathsFile> <range> <dest>"); System.exit(1);//from w w w. j a v a 2s .c o m } final String hadoopConfDir = IndexEnv.getHadoopConfDir(); final List<String> copyList = IndexEnv.getPathsRange(args[0], args[1]); if (copyList.isEmpty()) { log.error("No files to copy given {} {}", args[0], args[1]); System.exit(1); } DataConfig dataConfig = DataConfig.load(); SparkConf sparkConf = new SparkConf().setAppName("webindex-copy"); try (JavaSparkContext ctx = new JavaSparkContext(sparkConf)) { FileSystem hdfs = FileSystem.get(ctx.hadoopConfiguration()); Path destPath = new Path(args[2]); if (!hdfs.exists(destPath)) { hdfs.mkdirs(destPath); } log.info("Copying {} files (Range {} of paths file {}) from AWS to HDFS {}", copyList.size(), args[1], args[0], destPath.toString()); JavaRDD<String> copyRDD = ctx.parallelize(copyList, dataConfig.getNumExecutorInstances()); final String prefix = DataConfig.CC_URL_PREFIX; final String destDir = destPath.toString(); copyRDD.foreachPartition(iter -> { FileSystem fs = IndexEnv.getHDFS(hadoopConfDir); iter.forEachRemaining(ccPath -> { try { Path dfsPath = new Path(destDir + "/" + getFilename(ccPath)); if (fs.exists(dfsPath)) { log.error("File {} exists in HDFS and should have been previously filtered", dfsPath.getName()); } else { String urlToCopy = prefix + ccPath; log.info("Starting copy of {} to {}", urlToCopy, destDir); try (OutputStream out = fs.create(dfsPath); BufferedInputStream in = new BufferedInputStream( new URL(urlToCopy).openStream())) { IOUtils.copy(in, out); } log.info("Created {}", dfsPath.getName()); } } catch (IOException e) { log.error("Exception while copying {}", ccPath, e); } }); }); } }