List of usage examples for java.net URLConnection connect
public abstract void connect() throws IOException;
From source file:org.apache.tez.engine.common.shuffle.impl.Fetcher.java
/** * The connection establishment is attempted multiple times and is given up * only on the last failure. Instead of connecting with a timeout of * X, we try connecting with a timeout of x < X but multiple times. *///from w w w . java 2s .c om private void connect(URLConnection connection, int connectionTimeout) throws IOException { int unit = 0; if (connectionTimeout < 0) { throw new IOException("Invalid timeout " + "[timeout = " + connectionTimeout + " ms]"); } else if (connectionTimeout > 0) { unit = Math.min(UNIT_CONNECT_TIMEOUT, connectionTimeout); } // set the connect timeout to the unit-connect-timeout connection.setConnectTimeout(unit); while (true) { try { connection.connect(); break; } catch (IOException ioe) { // update the total remaining connect-timeout connectionTimeout -= unit; // throw an exception if we have waited for timeout amount of time // note that the updated value if timeout is used here if (connectionTimeout == 0) { throw ioe; } // reset the connect timeout for the last try if (connectionTimeout < unit) { unit = connectionTimeout; // reset the connect time out for the final connect connection.setConnectTimeout(unit); } } } }
From source file:android.webkit.cts.CtsTestServer.java
/** * Terminate the http server./*from w ww . j ava 2s . co m*/ */ public void shutdown() { try { // Avoid a deadlock between two threads where one is trying to call // close() and the other one is calling accept() by sending a GET // request for shutdown and having the server's one thread // sequentially call accept() and close(). URL url = new URL(mServerUri + SHUTDOWN_PREFIX); URLConnection connection = openConnection(url); connection.connect(); // Read the input from the stream to send the request. InputStream is = connection.getInputStream(); is.close(); // Block until the server thread is done shutting down. mServerThread.join(); } catch (MalformedURLException e) { throw new IllegalStateException(e); } catch (InterruptedException e) { throw new RuntimeException(e); } catch (IOException e) { throw new RuntimeException(e); } catch (NoSuchAlgorithmException e) { throw new IllegalStateException(e); } catch (KeyManagementException e) { throw new IllegalStateException(e); } }
From source file:com.google.code.twisty.Twisty.java
void startTerp(Uri gameURI) throws IOException, MalformedURLException { /* Set up output file in same directory as saved-games. */ String dir = getSavedGamesDir(true); if (dir == null) { showDialog(DIALOG_CANT_SAVE);//w w w . j ava 2 s. c o m return; } String uriString = gameURI.toString(); String gameFilename = uriString.substring(uriString.lastIndexOf("/") + 1); File outputFile = new File(dir, gameFilename); // Copy the input file into our story directory, unless the input and output file // would be the same. This check currently doesn't support uri's that don't contain // the file path. In that case the file will be corrupted. if (!((gameURI.getScheme().equals("content") || gameURI.getScheme().equals("file")) && gameURI.toString().contains(outputFile.getCanonicalPath()))) { FileOutputStream gameOutputStream = null; try { outputFile.createNewFile(); gameOutputStream = new FileOutputStream(outputFile); } catch (IOException e) { Log.i(TAG, "Failed to create file called " + gameFilename); return; } /* Set up input from URI */ InputStream gameInputStream = null; if (gameURI.getScheme().equals("content")) { try { gameInputStream = getContentResolver().openInputStream(gameURI); } catch (FileNotFoundException e) { Log.i(TAG, "Failed to open file: " + uriString); gameOutputStream.close(); return; } } else { try { URL gameURL = new URL(uriString); URLConnection connection = gameURL.openConnection(); connection.connect(); gameInputStream = connection.getInputStream(); } catch (MalformedURLException e) { Log.i(TAG, "Received malformed URI: " + uriString); gameOutputStream.close(); return; } catch (IOException e) { Log.i(TAG, "Failed to open connection to URI: " + uriString); gameOutputStream.close(); return; } } try { Log.i(TAG, "About to spew raw data to disk..."); suckstream(gameInputStream, gameOutputStream); } catch (IOException e) { Log.i(TAG, "Failed to copy URL contents to local file: " + uriString); return; } Log.i(TAG, "Completed dump of raw data to disk."); } else Log.i(TAG, "Input and output file are the same: " + outputFile.getCanonicalPath()); Log.i(TAG, "Starting gamefile located at " + outputFile.getCanonicalPath()); startTerp(outputFile.getCanonicalPath()); }
From source file:tm.alashow.datmusic.ui.activity.MainActivity.java
private void search(String query, final boolean refresh, long captchaSid, String captchaKey, boolean performerOnly) { oldQuery = query;//from w w w .j av a 2s . com RequestParams params = new RequestParams(); params.put("q", query); params.put("autocomplete", Config.VK_CONFIG_AUTOCOMPLETE); params.put("sort", CONFIG_SORT); params.put("count", CONFIG_COUNT); params.put("performer_only", (performerOnly) ? 1 : CONFIG_PERFORMER_ONLY); if (captchaSid > 1) { params.put("captcha_sid", captchaSid); params.put("captcha_key", captchaKey); } //change search method to getPopular, if query empty. get popular music. ApiClient.get(Config.SEARCH, params, new JsonHttpResponseHandler() { @Override public void onStart() { U.hideView(errorView); if (refresh) { swipeRefreshLayout.setRefreshing(true); } else { U.showView(progressBar); U.hideView(mListView); } } @Override public void onSuccess(int statusCode, Header[] headers, JSONObject response) { try { clearList();//clear old data if (response.has("error")) { //if we have error //Result errors JSONObject errorObject = response.getJSONObject("error"); int errorCode = errorObject.getInt("error_code"); //showing error switch (errorCode) { case 5: showError("token"); break; case 6: // "Too many requests per second" error, retry search(oldQuery); break; case 14: showCaptcha(errorObject.getString("captcha_img"), errorObject.getLong("captcha_sid")); showError("captcha"); break; default: showError(errorObject.getString("error_msg")); break; } return; } JSONArray audios = response.getJSONArray("response"); if (audios.length() >= 2) { for (int i = 1; i < audios.length(); i++) audioList.add(new Audio((JSONObject) audios.get(i))); audioListAdapter = new AudioListAdapter(MainActivity.this, audioList); mListView.setAdapter(audioListAdapter); mListView.setFastScrollEnabled(audioList.size() > 10); mListView.setOnItemClickListener(new AdapterView.OnItemClickListener() { @Override public void onItemClick(AdapterView<?> parent, View view, int position, long id) { final Audio audio = audioList.get(position); final BottomSheet bottomSheet = new BottomSheet.Builder(MainActivity.this) .title(audio.getArtist() + " - " + audio.getTitle()) .sheet(R.menu.audio_actions) .listener(new DialogInterface.OnClickListener() { @Override public void onClick(DialogInterface dialog, int which) { switch (which) { case R.id.download: DownloadManager mgr = (DownloadManager) getSystemService( DOWNLOAD_SERVICE); Uri downloadUri = Uri.parse(audio.getDownloadUrl()); DownloadManager.Request request = new DownloadManager.Request( downloadUri); if (U.isAboveOfVersion(11)) { request.setNotificationVisibility( DownloadManager.Request.VISIBILITY_VISIBLE_NOTIFY_COMPLETED); } request.setAllowedNetworkTypes( DownloadManager.Request.NETWORK_WIFI | DownloadManager.Request.NETWORK_MOBILE); mgr.enqueue(request); break; case R.id.play: playAudio(audio); break; case R.id.copy: if (!U.isAboveOfVersion(11)) { android.text.ClipboardManager clipboard = (android.text.ClipboardManager) getSystemService( CLIPBOARD_SERVICE); clipboard.setText(audio.getSecureDownloadUrl()); } else { android.content.ClipboardManager clipboard = (android.content.ClipboardManager) getSystemService( CLIPBOARD_SERVICE); android.content.ClipData clip = android.content.ClipData .newPlainText("Link", audio.getSecureDownloadUrl()); clipboard.setPrimaryClip(clip); U.showCenteredToast(MainActivity.this, R.string.audio_copied); } break; case R.id.share: String shareText = getString(R.string.share_text) + audio.getSecureDownloadUrl(); Intent sendIntent = new Intent(); sendIntent.setAction(Intent.ACTION_SEND); sendIntent.putExtra(Intent.EXTRA_TEXT, shareText); sendIntent.setType("text/plain"); startActivity(sendIntent); } } }).show(); //If file size already set, show it if (audio.getBytes() > 1) { setSizeAndBitrate(bottomSheet, audio); } else { try { new Thread(new Runnable() { @Override public void run() { try { URLConnection ucon; final URL uri = new URL(audio.getDownloadUrl()); ucon = uri.openConnection(); ucon.connect(); final long bytes = Long .parseLong(ucon.getHeaderField("content-length")); runOnUiThread(new Runnable() { @Override public void run() { audio.setBytes(bytes); setSizeAndBitrate(bottomSheet, audio); } }); } catch (Exception e) { e.printStackTrace(); } } }).start(); } catch (final Exception e) { e.printStackTrace(); } } } }); if (refresh) { swipeRefreshLayout.setRefreshing(false); } else { U.hideView(progressBar); } } else { showError("notFound"); } } catch (Exception e) { U.showCenteredToast(MainActivity.this, R.string.exception); e.printStackTrace(); } } @Override public void onFailure(int statusCode, Header[] headers, String responseString, Throwable throwable) { showError("network"); } @Override public void onFailure(int statusCode, Header[] headers, Throwable throwable, JSONObject errorResponse) { showError("network"); } @Override public void onFailure(int statusCode, Header[] headers, Throwable throwable, JSONArray errorResponse) { showError("network"); } @Override public void onFinish() { U.showView(mListView); if (refresh) { swipeRefreshLayout.setRefreshing(false); } else { U.hideView(progressBar); } } }); }
From source file:org.openqa.selenium.server.SeleniumServer.java
protected void readUserCommands() throws IOException { Sleeper.sleepTight(500);/*www . ja v a 2 s. c om*/ System.out.println( "Entering interactive mode... type Selenium commands here (e.g: cmd=open&1=http://www.yahoo.com)"); BufferedReader stdIn = new BufferedReader(new InputStreamReader(System.in)); String userInput; final String[] lastSessionId = new String[] { "" }; while ((userInput = stdIn.readLine()) != null) { userInput = userInput.trim(); if ("exit".equals(userInput) || "quit".equals(userInput)) { System.out.println("Stopping..."); stop(); System.exit(0); } if ("".equals(userInput)) continue; if (!userInput.startsWith("cmd=") && !userInput.startsWith("commandResult=")) { System.err.println("ERROR - Invalid command: \"" + userInput + "\""); continue; } final boolean newBrowserSession = userInput.contains("getNewBrowserSession"); if (!userInput.contains("sessionId") && !newBrowserSession) { userInput = userInput + "&sessionId=" + lastSessionId[0]; } final URL url = new URL( "http://localhost:" + configuration.getPort() + "/selenium-server/driver?" + userInput); Thread t = new Thread(new Runnable() { // Thread safety reviewed public void run() { InputStream is = null; try { LOGGER.info("---> Requesting " + url.toString()); URLConnection conn = url.openConnection(); conn.connect(); is = conn.getInputStream(); ByteArrayOutputStream out = new ByteArrayOutputStream(); byte[] buffer = new byte[2048]; int length; while ((length = is.read(buffer)) != -1) { out.write(buffer, 0, length); } is.close(); String output = out.toString(); if (newBrowserSession) { if (output.startsWith("OK,")) { lastSessionId[0] = output.substring(3); } } } catch (IOException e) { System.err.println(e.getMessage()); if (debugMode) { e.printStackTrace(); } } finally { if (is != null) { try { is.close(); } catch (IOException e) { System.err.println(e.getMessage()); if (debugMode) { e.printStackTrace(); } } } } } }); t.start(); } }
From source file:eu.semlibproject.annotationserver.restapis.ServicesAPI.java
/** * Implement a simple proxy// ww w . j a va 2 s . c om * * @param requestedURL the requested URL * @param req the HttpServletRequest * @return */ @GET @Path("proxy") public Response proxy(@QueryParam(SemlibConstants.URL_PARAM) String requestedURL, @Context HttpServletRequest req) { BufferedReader in = null; try { URL url = new URL(requestedURL); URLConnection urlConnection = url.openConnection(); int proxyConnectionTimeout = ConfigManager.getInstance().getProxyAPITimeout(); // Set base properties urlConnection.setUseCaches(false); urlConnection.setConnectTimeout(proxyConnectionTimeout * 1000); // set max response timeout 15 sec String acceptedDataFormat = req.getHeader(SemlibConstants.HTTP_HEADER_ACCEPT); if (StringUtils.isNotBlank(acceptedDataFormat)) { urlConnection.addRequestProperty(SemlibConstants.HTTP_HEADER_ACCEPT, acceptedDataFormat); } // Open the connection urlConnection.connect(); if (urlConnection instanceof HttpURLConnection) { HttpURLConnection httpConnection = (HttpURLConnection) urlConnection; int statusCode = httpConnection.getResponseCode(); if (statusCode == HttpURLConnection.HTTP_MOVED_TEMP || statusCode == HttpURLConnection.HTTP_MOVED_PERM) { // Follow the redirect String newLocation = httpConnection.getHeaderField(SemlibConstants.HTTP_HEADER_LOCATION); httpConnection.disconnect(); if (StringUtils.isNotBlank(newLocation)) { return this.proxy(newLocation, req); } else { return Response.status(statusCode).build(); } } else if (statusCode == HttpURLConnection.HTTP_OK) { // Send the response StringBuilder sbf = new StringBuilder(); // Check if the contentType is supported boolean contentTypeSupported = false; String contentType = httpConnection.getHeaderField(SemlibConstants.HTTP_HEADER_CONTENT_TYPE); List<String> supportedMimeTypes = ConfigManager.getInstance().getProxySupportedMimeTypes(); if (contentType != null) { for (String cMime : supportedMimeTypes) { if (contentType.equals(cMime) || contentType.contains(cMime)) { contentTypeSupported = true; break; } } } if (!contentTypeSupported) { httpConnection.disconnect(); return Response.status(Status.NOT_ACCEPTABLE).build(); } String contentEncoding = httpConnection.getContentEncoding(); if (StringUtils.isBlank(contentEncoding)) { contentEncoding = "UTF-8"; } InputStreamReader inStrem = new InputStreamReader((InputStream) httpConnection.getContent(), Charset.forName(contentEncoding)); in = new BufferedReader(inStrem); String inputLine; while ((inputLine = in.readLine()) != null) { sbf.append(inputLine); sbf.append("\r\n"); } in.close(); httpConnection.disconnect(); return Response.status(statusCode).header(SemlibConstants.HTTP_HEADER_CONTENT_TYPE, contentType) .entity(sbf.toString()).build(); } else { httpConnection.disconnect(); return Response.status(statusCode).build(); } } return Response.status(Status.BAD_REQUEST).build(); } catch (MalformedURLException ex) { logger.log(Level.SEVERE, null, ex); return Response.status(Status.BAD_REQUEST).build(); } catch (IOException ex) { logger.log(Level.SEVERE, null, ex); return Response.status(Status.INTERNAL_SERVER_ERROR).build(); } finally { if (in != null) { try { in.close(); } catch (IOException ex) { logger.log(Level.SEVERE, null, ex); return Response.status(Status.INTERNAL_SERVER_ERROR).build(); } } } }
From source file:com.redhat.rhn.frontend.action.common.DownloadFile.java
@Override protected StreamInfo getStreamInfo(ActionMapping mapping, ActionForm form, HttpServletRequest request, HttpServletResponse response) throws Exception { String path = ""; Map params = (Map) request.getAttribute(PARAMS); String type = (String) params.get(TYPE); if (type.equals(DownloadManager.DOWNLOAD_TYPE_KICKSTART)) { return getStreamInfoKickstart(mapping, form, request, response, path); } else if (type.equals(DownloadManager.DOWNLOAD_TYPE_COBBLER)) { String url = ConfigDefaults.get().getCobblerServerUrl() + (String) params.get(URL_STRING); KickstartHelper helper = new KickstartHelper(request); String data = ""; if (helper.isProxyRequest()) { data = KickstartManager.getInstance().renderKickstart(helper.getKickstartHost(), url); } else {//from w w w. j av a 2 s . c o m data = KickstartManager.getInstance().renderKickstart(url); } setTextContentInfo(response, data.length()); return getStreamForText(data.getBytes()); } else if (type.equals(DownloadManager.DOWNLOAD_TYPE_COBBLER_API)) { // read data from POST body String postData = new String(); String line = null; BufferedReader reader = request.getReader(); while ((line = reader.readLine()) != null) { postData += line; } // Send data URL url = new URL(ConfigDefaults.get().getCobblerServerUrl() + "/cobbler_api"); URLConnection conn = url.openConnection(); conn.setDoOutput(true); OutputStreamWriter wr = new OutputStreamWriter(conn.getOutputStream()); // this will write POST /download//cobbler_api instead of // POST /cobbler_api, but cobbler do not mind wr.write(postData, 0, postData.length()); wr.flush(); conn.connect(); // Get the response String output = new String(); BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream())); while ((line = rd.readLine()) != null) { output += line; } wr.close(); KickstartHelper helper = new KickstartHelper(request); if (helper.isProxyRequest()) { // Search/replacing all instances of cobbler host with host // we pass in, for use with Spacewalk Proxy. output = output.replaceAll(ConfigDefaults.get().getCobblerHost(), helper.getForwardedHost()); } setXmlContentInfo(response, output.length()); return getStreamForXml(output.getBytes()); } else { Long fileId = (Long) params.get(FILEID); Long userid = (Long) params.get(USERID); User user = UserFactory.lookupById(userid); if (type.equals(DownloadManager.DOWNLOAD_TYPE_PACKAGE)) { Package pack = PackageFactory.lookupByIdAndOrg(fileId, user.getOrg()); setBinaryContentInfo(response, pack.getPackageSize().intValue()); path = Config.get().getString(ConfigDefaults.MOUNT_POINT) + "/" + pack.getPath(); return getStreamForBinary(path); } else if (type.equals(DownloadManager.DOWNLOAD_TYPE_SOURCE)) { Package pack = PackageFactory.lookupByIdAndOrg(fileId, user.getOrg()); List<PackageSource> src = PackageFactory.lookupPackageSources(pack); if (!src.isEmpty()) { setBinaryContentInfo(response, src.get(0).getPackageSize().intValue()); path = Config.get().getString(ConfigDefaults.MOUNT_POINT) + "/" + src.get(0).getPath(); return getStreamForBinary(path); } } else if (type.equals(DownloadManager.DOWNLOAD_TYPE_REPO_LOG)) { Channel c = ChannelFactory.lookupById(fileId); ChannelManager.verifyChannelAdmin(user, fileId); StringBuilder output = new StringBuilder(); for (String fileName : ChannelManager.getLatestSyncLogFiles(c)) { RandomAccessFile file = new RandomAccessFile(fileName, "r"); long fileLength = file.length(); if (fileLength > DOWNLOAD_REPO_LOG_LENGTH) { file.seek(fileLength - DOWNLOAD_REPO_LOG_LENGTH); // throw away text till end of the actual line file.readLine(); } else { file.seek(0); } String line; while ((line = file.readLine()) != null) { output.append(line); output.append("\n"); } file.close(); if (output.length() > DOWNLOAD_REPO_LOG_MIN_LENGTH) { break; } } setTextContentInfo(response, output.length()); return getStreamForText(output.toString().getBytes()); } else if (type.equals(DownloadManager.DOWNLOAD_TYPE_CRASHFILE)) { CrashFile crashFile = CrashManager.lookupCrashFileByUserAndId(user, fileId); String crashPath = crashFile.getCrash().getStoragePath(); setBinaryContentInfo(response, (int) crashFile.getFilesize()); path = Config.get().getString(ConfigDefaults.MOUNT_POINT) + "/" + crashPath + "/" + crashFile.getFilename(); return getStreamForBinary(path); } } throw new UnknownDownloadTypeException( "The specified download type " + type + " is not currently supported"); }
From source file:org.ecoinformatics.datamanager.download.DownloadHandler.java
/** * Gets content from given source and writes it to DataStorageInterface * to store them. This method will be called by run() * //www . j a v a2s . c o m * @param resourceName the URL to the source data to be retrieved */ protected boolean getContentFromSource(String resourceName) { boolean successFlag = false; QualityCheck onlineURLsQualityCheck = null; boolean onlineURLsException = false; // used to determine status of onlineURLs quality check if (resourceName != null) { resourceName = resourceName.trim(); } if (resourceName != null && (resourceName.startsWith("http://") || resourceName.startsWith("https://") || resourceName.startsWith("file://") || resourceName.startsWith("ftp://"))) { // get the data from a URL int responseCode = 0; String responseMessage = null; try { URL url = new URL(resourceName); boolean isFTP = false; if (entity != null) { String contentType = null; // Find the right MIME type and set it as content type if (resourceName.startsWith("http")) { HttpURLConnection httpURLConnection = (HttpURLConnection) url.openConnection(); httpURLConnection.setRequestMethod("HEAD"); httpURLConnection.connect(); contentType = httpURLConnection.getContentType(); responseCode = httpURLConnection.getResponseCode(); responseMessage = httpURLConnection.getResponseMessage(); } else if (resourceName.startsWith("file")) { URLConnection urlConnection = url.openConnection(); urlConnection.connect(); contentType = urlConnection.getContentType(); } else { // FTP isFTP = true; contentType = "application/octet-stream"; } entity.setUrlContentType(contentType); } if (!isFTP) { // HTTP(S) or FILE InputStream filestream = url.openStream(); try { successFlag = this.writeRemoteInputStreamIntoDataStorage(filestream); } catch (IOException e) { exception = e; String errorMessage = e.getMessage(); if (errorMessage.startsWith(ONLINE_URLS_EXCEPTION_MESSAGE)) { onlineURLsException = true; } } finally { filestream.close(); } } else { // FTP String[] urlParts = resourceName.split("/"); String address = urlParts[2]; String dir = "/"; for (int i = 3; i < urlParts.length - 1; i++) { dir += urlParts[i] + "/"; } String fileName = urlParts[urlParts.length - 1]; FTPClient ftpClient = new FTPClient(); ftpClient.connect(address); ftpClient.login(ANONYMOUS, anonymousFtpPasswd); ftpClient.changeWorkingDirectory(dir); ftpClient.setFileType(FTP.BINARY_FILE_TYPE); ftpClient.enterLocalPassiveMode(); // necessary to avoid firewall blocking InputStream filestream = ftpClient.retrieveFileStream(fileName); try { successFlag = this.writeRemoteInputStreamIntoDataStorage(filestream); } catch (IOException e) { exception = e; String errorMessage = e.getMessage(); if (errorMessage.startsWith(ONLINE_URLS_EXCEPTION_MESSAGE)) { onlineURLsException = true; } } finally { try { filestream.close(); } catch (IOException e) { exception = new DataSourceNotFoundException(String .format("Error closing local file '%s': %s", resourceName, e.getMessage())); onlineURLsException = true; } } // logout and disconnect if FTP session if (resourceName.startsWith("ftp") && ftpClient != null) { try { ftpClient.enterLocalActiveMode(); ftpClient.logout(); ftpClient.disconnect(); } catch (IOException e) { exception = new DataSourceNotFoundException( String.format("Error disconnecting from FTP with resource '%s': %s", resourceName, e.getMessage())); onlineURLsException = true; } } } } catch (MalformedURLException e) { String eClassName = e.getClass().getName(); String eMessage = String.format("%s: %s", eClassName, e.getMessage()); exception = new DataSourceNotFoundException( String.format("The URL '%s' is a malformed URL: %s", resourceName, eMessage)); } catch (IOException e) { String eClassName = e.getClass().getName(); String eMessage = String.format("%s: %s", eClassName, e.getMessage()); if (responseCode > 0) { eMessage = String.format("Response Code: %d %s; %s", responseCode, responseMessage, eMessage); } exception = new DataSourceNotFoundException( String.format("The URL '%s' is not reachable: %s", resourceName, eMessage)); } // Initialize the "Online URLs are live" quality check String qualityCheckIdentifier = "onlineURLs"; QualityCheck qualityCheckTemplate = QualityReport.getQualityCheckTemplate(qualityCheckIdentifier); onlineURLsQualityCheck = new QualityCheck(qualityCheckIdentifier, qualityCheckTemplate); if (QualityCheck.shouldRunQualityCheck(entity, onlineURLsQualityCheck)) { String resourceNameEscaped = embedInCDATA(resourceName); if (!onlineURLsException) { onlineURLsQualityCheck.setStatus(Status.valid); onlineURLsQualityCheck.setFound("true"); onlineURLsQualityCheck.setExplanation("Succeeded in accessing URL: " + resourceNameEscaped); } else { onlineURLsQualityCheck.setFailedStatus(); onlineURLsQualityCheck.setFound("false"); String explanation = "Failed to access URL: " + resourceNameEscaped; explanation = explanation + "; " + embedInCDATA(exception.getMessage()); onlineURLsQualityCheck.setExplanation(explanation); } entity.addQualityCheck(onlineURLsQualityCheck); } return successFlag; } else if (resourceName != null && resourceName.startsWith("ecogrid://")) { // get the docid from url int start = resourceName.indexOf("/", 11) + 1; //log.debug("start: " + start); int end = resourceName.indexOf("/", start); if (end == -1) { end = resourceName.length(); } //log.debug("end: " + end); String ecogridIdentifier = resourceName.substring(start, end); // pass this docid and get data item //System.out.println("the endpoint is "+ECOGRIDENDPOINT); //System.out.println("The identifier is "+ecogridIdentifier); //return false; return getContentFromEcoGridSource(ecogridEndPoint, ecogridIdentifier); } else if (resourceName != null && resourceName.startsWith("srb://")) { // get srb docid from the url String srbIdentifier = transformSRBurlToDocid(resourceName); // reset endpoint for srb (This is hack we need to figure ou // elegent way to do this //mEndPoint = Config.getValue("//ecogridService/srb/endPoint"); // pass this docid and get data item //log.debug("before get srb data"); return getContentFromEcoGridSource(SRBENDPOINT, srbIdentifier); } else { successFlag = false; return successFlag; } }
From source file:edu.lternet.pasta.dml.download.DownloadHandler.java
/** * Gets content from given source and writes it to DataStorageInterface * to store them. This method will be called by run() * //from w w w. ja v a 2 s . c om * @param resourceName the URL to the source data to be retrieved */ protected boolean getContentFromSource(String resourceName) { boolean successFlag = false; QualityCheck onlineURLsQualityCheck = null; boolean onlineURLsException = false; // used to determine status of onlineURLs quality check if (resourceName != null) { resourceName = resourceName.trim(); } if (resourceName != null && (resourceName.startsWith("http://") || resourceName.startsWith("https://") || resourceName.startsWith("file://") || resourceName.startsWith("ftp://"))) { // get the data from a URL int responseCode = 0; String responseMessage = null; try { URL url = new URL(resourceName); boolean isFTP = false; if (entity != null) { String contentType = null; // Find the right MIME type and set it as content type if (resourceName.startsWith("http")) { HttpURLConnection httpURLConnection = (HttpURLConnection) url.openConnection(); httpURLConnection.setRequestMethod("HEAD"); httpURLConnection.connect(); contentType = httpURLConnection.getContentType(); responseCode = httpURLConnection.getResponseCode(); responseMessage = httpURLConnection.getResponseMessage(); } else if (resourceName.startsWith("file")) { URLConnection urlConnection = url.openConnection(); urlConnection.connect(); contentType = urlConnection.getContentType(); } else { // FTP isFTP = true; contentType = "application/octet-stream"; } entity.setUrlContentType(contentType); } if (!isFTP) { // HTTP(S) or FILE InputStream filestream = url.openStream(); try { successFlag = this.writeRemoteInputStreamIntoDataStorage(filestream); } catch (IOException e) { exception = e; String errorMessage = e.getMessage(); if (errorMessage.startsWith(ONLINE_URLS_EXCEPTION_MESSAGE)) { onlineURLsException = true; } } finally { filestream.close(); } } else { // FTP String[] urlParts = resourceName.split("/"); String address = urlParts[2]; String dir = "/"; for (int i = 3; i < urlParts.length - 1; i++) { dir += urlParts[i] + "/"; } String fileName = urlParts[urlParts.length - 1]; FTPClient ftpClient = new FTPClient(); ftpClient.connect(address); ftpClient.login(ANONYMOUS, anonymousFtpPasswd); ftpClient.changeWorkingDirectory(dir); ftpClient.setFileType(FTP.BINARY_FILE_TYPE); ftpClient.enterLocalPassiveMode(); // necessary to avoid firewall blocking InputStream filestream = ftpClient.retrieveFileStream(fileName); try { successFlag = this.writeRemoteInputStreamIntoDataStorage(filestream); } catch (IOException e) { exception = e; String errorMessage = e.getMessage(); if (errorMessage.startsWith(ONLINE_URLS_EXCEPTION_MESSAGE)) { onlineURLsException = true; } } finally { try { filestream.close(); } catch (IOException e) { exception = new DataSourceNotFoundException(String .format("Error closing local file '%s': %s", resourceName, e.getMessage())); onlineURLsException = true; } } // logout and disconnect if FTP session if (resourceName.startsWith("ftp") && ftpClient != null) { try { ftpClient.enterLocalActiveMode(); ftpClient.logout(); ftpClient.disconnect(); } catch (IOException e) { exception = new DataSourceNotFoundException( String.format("Error disconnecting from FTP with resource '%s': %s", resourceName, e.getMessage())); onlineURLsException = true; } } } } catch (MalformedURLException e) { String eClassName = e.getClass().getName(); String eMessage = String.format("%s: %s", eClassName, e.getMessage()); onlineURLsException = true; exception = new DataSourceNotFoundException( String.format("The URL '%s' is a malformed URL: %s", resourceName, eMessage)); } catch (IOException e) { String eClassName = e.getClass().getName(); String eMessage = String.format("%s: %s", eClassName, e.getMessage()); if (responseCode > 0) { eMessage = String.format("Response Code: %d %s; %s", responseCode, responseMessage, eMessage); } onlineURLsException = true; exception = new DataSourceNotFoundException( String.format("The URL '%s' is not reachable: %s", resourceName, eMessage)); } // Initialize the "Online URLs are live" quality check String qualityCheckIdentifier = "onlineURLs"; QualityCheck qualityCheckTemplate = QualityReport.getQualityCheckTemplate(qualityCheckIdentifier); onlineURLsQualityCheck = new QualityCheck(qualityCheckIdentifier, qualityCheckTemplate); if (QualityCheck.shouldRunQualityCheck(entity, onlineURLsQualityCheck)) { String resourceNameEscaped = embedInCDATA(resourceName); if (!onlineURLsException) { onlineURLsQualityCheck.setStatus(Status.valid); onlineURLsQualityCheck.setFound("true"); onlineURLsQualityCheck.setExplanation("Succeeded in accessing URL: " + resourceNameEscaped); } else { onlineURLsQualityCheck.setFailedStatus(); onlineURLsQualityCheck.setFound("false"); String explanation = "Failed to access URL: " + resourceNameEscaped; explanation = explanation + "; " + embedInCDATA(exception.getMessage()); onlineURLsQualityCheck.setExplanation(explanation); } entity.addQualityCheck(onlineURLsQualityCheck); } return successFlag; } else if (resourceName != null && resourceName.startsWith("ecogrid://")) { // get the docid from url int start = resourceName.indexOf("/", 11) + 1; //log.debug("start: " + start); int end = resourceName.indexOf("/", start); if (end == -1) { end = resourceName.length(); } //log.debug("end: " + end); String ecogridIdentifier = resourceName.substring(start, end); // pass this docid and get data item //System.out.println("the endpoint is "+ECOGRIDENDPOINT); //System.out.println("The identifier is "+ecogridIdentifier); //return false; return getContentFromEcoGridSource(ecogridEndPoint, ecogridIdentifier); } else if (resourceName != null && resourceName.startsWith("srb://")) { // get srb docid from the url String srbIdentifier = transformSRBurlToDocid(resourceName); // reset endpoint for srb (This is hack we need to figure ou // elegent way to do this //mEndPoint = Config.getValue("//ecogridService/srb/endPoint"); // pass this docid and get data item //log.debug("before get srb data"); return getContentFromEcoGridSource(SRBENDPOINT, srbIdentifier); } else { successFlag = false; return successFlag; } }