List of usage examples for java.net HttpURLConnection getResponseMessage
public String getResponseMessage() throws IOException
From source file:easyshop.downloadhelper.HttpPageGetter.java
public HttpPage getDHttpPage(PageRef url, String charSet) { count++;//www. j ava 2 s . co m log.debug("getURL(" + count + ")"); if (url.getUrlStr() == null) { ConnResponse conRes = new ConnResponse(null, null, 0, 0, 0); return new OriHttpPage(-1, null, null, null, conRes, null); } URL requestedURL = null; try { requestedURL = new URL(url.getUrlStr()); } catch (MalformedURLException e1) { // TODO Auto-generated catch block log.error("wrong urlstr" + url.getUrlStr()); ConnResponse conRes = new ConnResponse(null, null, 0, 0, 0); return new OriHttpPage(-1, null, null, null, conRes, null); } ; // System.out.println(""+requestedURL.toExternalForm()); URL referer = null; try { log.debug("Creating HTTP connection to " + requestedURL); HttpURLConnection conn = (HttpURLConnection) requestedURL.openConnection(); if (referer != null) { log.debug("Setting Referer header to " + referer); conn.setRequestProperty("Referer", referer.toExternalForm()); } if (userAgent != null) { log.debug("Setting User-Agent to " + userAgent); conn.setRequestProperty("User-Agent", userAgent); } // DateFormat dateFormat=DateFormat.getDateInstance(); // conn.setRequestProperty("If-Modlfied-Since",dateFormat.parse("2005-08-15 20:18:30").toGMTString()); conn.setUseCaches(false); // conn.setRequestProperty("connection","keep-alive"); for (Iterator it = conn.getRequestProperties().keySet().iterator(); it.hasNext();) { String key = (String) it.next(); if (key == null) { break; } String value = conn.getHeaderField(key); // System.out.println("Request header " + key + ": " + value); } log.debug("Opening URL"); long startTime = System.currentTimeMillis(); conn.connect(); String resp = conn.getResponseMessage(); log.debug("Remote server response: " + resp); int code = conn.getResponseCode(); if (code != 200) { log.error("Could not get connection for code=" + code); System.err.println("Could not get connection for code=" + code); ConnResponse conRes = new ConnResponse(null, null, 0, 0, code); return new HttpPage(requestedURL.toExternalForm(), null, conRes, null); } // if (conn.getContentLength()<=0||conn.getContentLength()>10000000){ // log.error("Content length==0"); // System.err.println("Content length==0"); // ConnResponse conRes=new ConnResponse(null,null,null,0,0,-100); // return new URLObject(-1,requestedURL, null,null,conRes); // } String respStr = conn.getHeaderField(0); long serverDate = conn.getDate(); // log.info("Server response: " + respStr); for (int i = 1; i < conn.getHeaderFields().size(); i++) { String key = conn.getHeaderFieldKey(i); if (key == null) { break; } String value = conn.getHeaderField(key); // System.out.println("Received header " + key + ": " + value); // log.debug("Received header " + key + ": " + value); } // log.debug("Getting buffered input stream from remote connection"); log.debug("start download(" + count + ")"); BufferedInputStream remoteBIS = new BufferedInputStream(conn.getInputStream()); ByteArrayOutputStream baos = new ByteArrayOutputStream(10240); byte[] buf = new byte[1024]; int bytesRead = 0; while (bytesRead >= 0) { baos.write(buf, 0, bytesRead); bytesRead = remoteBIS.read(buf); } // baos.write(remoteBIS.read(new byte[conn.getContentLength()])); // remoteBIS.close(); byte[] content = baos.toByteArray(); long timeTaken = System.currentTimeMillis() - startTime; if (timeTaken < 100) timeTaken = 500; int bytesPerSec = (int) ((double) content.length / ((double) timeTaken / 1000.0)); // log.info("Downloaded " + content.length + " bytes, " + bytesPerSec + " bytes/sec"); if (content.length < conn.getContentLength()) { log.warn("Didn't download full content for URL: " + url); // failureCount++; ConnResponse conRes = new ConnResponse(conn.getContentType(), null, content.length, serverDate, code); return new HttpPage(requestedURL.toExternalForm(), null, conRes, conn.getContentType()); } log.debug("download(" + count + ")"); ConnResponse conRes = new ConnResponse(conn.getContentType(), null, conn.getContentLength(), serverDate, code); String c = charSet; if (c == null) c = conRes.getCharSet(); HttpPage obj = new HttpPage(requestedURL.toExternalForm(), content, conRes, c); return obj; } catch (IOException ioe) { log.warn("Caught IO Exception: " + ioe.getMessage(), ioe); failureCount++; ConnResponse conRes = new ConnResponse(null, null, 0, 0, 0); return new HttpPage(requestedURL.toExternalForm(), null, conRes, null); } catch (Exception e) { log.warn("Caught Exception: " + e.getMessage(), e); failureCount++; ConnResponse conRes = new ConnResponse(null, null, 0, 0, 0); return new HttpPage(requestedURL.toExternalForm(), null, conRes, null); } }
From source file:net.myrrix.client.ClientRecommender.java
private void refreshPartition(int partition) { String urlPath = "/refresh"; for (HostAndPort replica : partitions.get(partition)) { HttpURLConnection connection = null; try {//from w ww .j av a2 s . c o m connection = buildConnectionToReplica(replica, urlPath, "POST"); if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) { log.warn("Unable to refresh partition {} ({} {}); continuing", partition, connection.getResponseCode(), connection.getResponseMessage()); // Yes, continue } } catch (IOException ioe) { log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString()); } finally { if (connection != null) { connection.disconnect(); } } } }
From source file:org.runnerup.export.NikePlusSynchronizer.java
@Override public Status upload(SQLiteDatabase db, long mID) { Status s;//from w w w . j a v a 2 s.c o m if ((s = connect()) != Status.OK) { return s; } NikeXML nikeXML = new NikeXML(db); GPX nikeGPX = new GPX(db); HttpURLConnection conn = null; Exception ex = null; try { StringWriter xml = new StringWriter(); nikeXML.export(mID, xml); StringWriter gpx = new StringWriter(); nikeGPX.export(mID, gpx); String url = String.format(SYNC_URL, access_token); conn = (HttpURLConnection) new URL(url).openConnection(); conn.setDoOutput(true); conn.setRequestMethod(RequestMethod.POST.name()); conn.addRequestProperty("user-agent", USER_AGENT); conn.addRequestProperty("appid", APP_ID); Part<StringWritable> part1 = new Part<StringWritable>("runXML", new StringWritable(xml.toString())); part1.setFilename("runXML.xml"); part1.setContentType("text/plain; charset=US-ASCII"); part1.setContentTransferEncoding("8bit"); Part<StringWritable> part2 = new Part<StringWritable>("gpxXML", new StringWritable(gpx.toString())); part2.setFilename("gpxXML.xml"); part2.setContentType("text/plain; charset=US-ASCII"); part2.setContentTransferEncoding("8bit"); Part<?> parts[] = { part1, part2 }; SyncHelper.postMulti(conn, parts); int responseCode = conn.getResponseCode(); String amsg = conn.getResponseMessage(); conn.connect(); if (responseCode != HttpStatus.SC_OK) { throw new Exception(amsg); } url = String.format(SYNC_COMPLETE_URL, access_token); conn = (HttpURLConnection) new URL(url).openConnection(); conn.setDoOutput(true); conn.setRequestMethod(RequestMethod.POST.name()); conn.addRequestProperty("user-agent", USER_AGENT); conn.addRequestProperty("appid", APP_ID); responseCode = conn.getResponseCode(); amsg = conn.getResponseMessage(); conn.disconnect(); if (responseCode == HttpStatus.SC_OK) { s = Status.OK; s.activityId = mID; return s; } ex = new Exception(amsg); } catch (Exception e) { ex = e; } s = Synchronizer.Status.ERROR; s.ex = ex; s.activityId = mID; if (ex != null) { ex.printStackTrace(); } return s; }
From source file:com.neoteric.starter.metrics.report.elastic.ElasticsearchReporter.java
private void checkForIndexTemplate() { try {/*from www .j a v a2 s . c om*/ HttpURLConnection connection = openConnection("/_template/metrics_template", "HEAD"); connection.disconnect(); boolean isTemplateMissing = connection.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND; // nothing there, lets create it if (isTemplateMissing) { LOGGER.debug("No metrics template found in elasticsearch. Adding..."); HttpURLConnection putTemplateConnection = openConnection("/_template/metrics_template", "PUT"); JsonGenerator json = new JsonFactory().createGenerator(putTemplateConnection.getOutputStream()); json.writeStartObject(); json.writeStringField("template", index + "*"); json.writeObjectFieldStart("mappings"); json.writeObjectFieldStart("_default_"); json.writeObjectFieldStart("_all"); json.writeBooleanField("enabled", false); json.writeEndObject(); json.writeObjectFieldStart("properties"); json.writeObjectFieldStart("name"); json.writeObjectField("type", "string"); json.writeObjectField("index", "not_analyzed"); json.writeEndObject(); json.writeEndObject(); json.writeEndObject(); json.writeEndObject(); json.writeEndObject(); json.flush(); putTemplateConnection.disconnect(); if (putTemplateConnection.getResponseCode() != HttpStatus.OK.value()) { LOGGER.error( "Error adding metrics template to elasticsearch: {}/{}" + putTemplateConnection.getResponseCode(), putTemplateConnection.getResponseMessage()); } } checkedForIndexTemplate = true; } catch (IOException e) { LOGGER.error("Error when checking/adding metrics template to elasticsearch", e); } }
From source file:com.bjorsond.android.timeline.sync.ServerUploader.java
protected static void uploadFile(String locationFilename, String saveFilename) { System.out.println("saving " + locationFilename + "!! "); if (!saveFilename.contains(".")) saveFilename = saveFilename + Utilities.getExtension(locationFilename); if (!fileExistsOnServer(saveFilename)) { HttpURLConnection connection = null; DataOutputStream outputStream = null; String pathToOurFile = locationFilename; String urlServer = "http://folk.ntnu.no/bjornava/upload/upload.php"; // String urlServer = "http://timelinegamified.appspot.com/upload.php"; String lineEnd = "\r\n"; String twoHyphens = "--"; String boundary = "*****"; int bytesRead, bytesAvailable, bufferSize; byte[] buffer; int maxBufferSize = 1 * 1024 * 1024 * 1024; try {/*from w w w . ja v a2 s .c o m*/ FileInputStream fileInputStream = new FileInputStream(new File(pathToOurFile)); URL url = new URL(urlServer); connection = (HttpURLConnection) url.openConnection(); // Allow Inputs & Outputs connection.setDoInput(true); connection.setDoOutput(true); connection.setUseCaches(false); // Enable POST method connection.setRequestMethod("POST"); connection.setRequestProperty("Connection", "Keep-Alive"); connection.setRequestProperty("Content-Type", "multipart/form-data;boundary=" + boundary); outputStream = new DataOutputStream(connection.getOutputStream()); outputStream.writeBytes(twoHyphens + boundary + lineEnd); outputStream.writeBytes("Content-Disposition: form-data; name=\"uploadedfile\";filename=\"" + saveFilename + "\"" + lineEnd); outputStream.writeBytes(lineEnd); bytesAvailable = fileInputStream.available(); bufferSize = Math.min(bytesAvailable, maxBufferSize); buffer = new byte[bufferSize]; // Read file bytesRead = fileInputStream.read(buffer, 0, bufferSize); while (bytesRead > 0) { outputStream.write(buffer, 0, bufferSize); bytesAvailable = fileInputStream.available(); bufferSize = Math.min(bytesAvailable, maxBufferSize); bytesRead = fileInputStream.read(buffer, 0, bufferSize); } outputStream.writeBytes(lineEnd); outputStream.writeBytes(twoHyphens + boundary + twoHyphens + lineEnd); // Responses from the server (code and message) int serverResponseCode = connection.getResponseCode(); String serverResponseMessage = connection.getResponseMessage(); fileInputStream.close(); outputStream.flush(); outputStream.close(); System.out.println("Server response: " + serverResponseCode + " Message: " + serverResponseMessage); } catch (Exception ex) { //Exception handling } } else { System.out.println("image exists on server"); } }
From source file:net.myrrix.client.ClientRecommender.java
/** * <p>Lists the items that were most influential in recommending a given item to a given user. Exactly how this * is determined is left to the implementation, but, generally this will return items that the user prefers * and that are similar to the given item.</p> * * <p>These values by which the results are ordered are opaque values and have no interpretation * other than that larger means stronger.</p> * * @param userID ID of user who was recommended the item * @param itemID ID of item that was recommended * @param howMany maximum number of items to return * @return {@link List} of {@link RecommendedItem}, ordered from most influential in recommended the given * item to least/*from www . j a v a 2 s . c om*/ * @throws NoSuchUserException if the user is not known in the model * @throws NoSuchItemException if the item is not known in the model * @throws NotReadyException if the recommender has no model available yet * @throws TasteException if another error occurs */ @Override public List<RecommendedItem> recommendedBecause(long userID, long itemID, int howMany) throws TasteException { String urlPath = "/because/" + userID + '/' + itemID + "?howMany=" + howMany; TasteException savedException = null; for (HostAndPort replica : choosePartitionAndReplicas(userID)) { HttpURLConnection connection = null; try { connection = buildConnectionToReplica(replica, urlPath, "GET"); switch (connection.getResponseCode()) { case HttpURLConnection.HTTP_OK: return consumeItems(connection); case HttpURLConnection.HTTP_NOT_FOUND: String connectionMessage = connection.getResponseMessage(); if (connectionMessage != null && connectionMessage.contains(NoSuchUserException.class.getSimpleName())) { throw new NoSuchUserException(userID); } else { throw new NoSuchItemException(itemID); } case HttpURLConnection.HTTP_UNAVAILABLE: throw new NotReadyException(); default: throw new TasteException(connection.getResponseCode() + " " + connection.getResponseMessage()); } } catch (TasteException te) { log.info("Can't access {} at {}: ({})", urlPath, replica, te.toString()); savedException = te; } catch (IOException ioe) { log.info("Can't access {} at {}: ({})", urlPath, replica, ioe.toString()); savedException = new TasteException(ioe); } finally { if (connection != null) { connection.disconnect(); } } } throw savedException; }
From source file:com.example.rtobase2.AppEngineClient.java
public static Response getOrPost(Request request) { mErrorMessage = null;/*from w w w. j ava 2s . co m*/ HttpURLConnection conn = null; Response response = null; try { conn = (HttpURLConnection) request.uri.openConnection(); // if (!mAuthenticator.authenticate(conn)) { // mErrorMessage = str(R.string.aerc_authentication_failed) + ": " + mAuthenticator.errorMessage(); // } else { if (request.headers != null) { for (String header : request.headers.keySet()) { for (String value : request.headers.get(header)) { conn.addRequestProperty(header, value); } } } if (request instanceof PUT) { byte[] payload = ((PUT) request).body; String s = new String(payload, "UTF-8"); conn.setDoOutput(true); conn.setDoInput(true); conn.setRequestMethod("PUT"); JSONObject jsonobj = getJSONObject(s); conn.setRequestProperty("Content-Type", "application/json; charset=utf8"); OutputStream os = conn.getOutputStream(); os.write(jsonobj.toString().getBytes("UTF-8")); os.close(); } if (request instanceof POST) { byte[] payload = ((POST) request).body; String s = new String(payload, "UTF-8"); conn.setDoOutput(true); conn.setDoInput(true); conn.setRequestMethod("POST"); JSONObject jsonobj = getJSONObject(s); conn.setRequestProperty("Content-Type", "application/json; charset=utf8"); // ... OutputStream os = conn.getOutputStream(); os.write(jsonobj.toString().getBytes("UTF-8")); os.close(); // conn.setFixedLengthStreamingMode(payload.length); // conn.getOutputStream().write(payload); int status = conn.getResponseCode(); if (status / 100 != 2) response = new Response(status, new Hashtable<String, List<String>>(), conn.getResponseMessage().getBytes()); } if (response == null) { BufferedInputStream in = new BufferedInputStream(conn.getInputStream()); byte[] body = readStream(in); response = new Response(conn.getResponseCode(), conn.getHeaderFields(), body); // List<String> a = conn.getHeaderFields().get("aa"); } } } catch (IOException e) { e.printStackTrace(System.err); mErrorMessage = ((request instanceof POST) ? "POST " : "GET ") + str(R.string.aerc_failed) + ": " + e.getLocalizedMessage(); } finally { if (conn != null) conn.disconnect(); } return response; }
From source file:edwardawebb.queueman.classes.NetFlix.java
/** * /*w ww . ja va 2 s . c o m*/ * @param queueType * @param maxResults * @return HttpStatusCOde or NF_ERROR_BAD_DEFAULT for exceptions */ public int getQueue(int queueType, String maxResults) { Log.d("NetFlix", "getQueue()>>>"); URL QueueUrl = null; QueueHandler myQueueHandler = null; int result = NF_ERROR_BAD_DEFAULT; if (maxResults.equals(QueueMan.ALL_TITLES_STRING)) { maxResults = "500"; } // addtional info to return String expanders = "?expand=synopsis,formats&max_results=" + maxResults; InputStream xml = null; try { switch (queueType) { case NetFlixQueue.QUEUE_TYPE_INSTANT: if (!NetFlix.instantQueue.isEmpty() && instantQueue.isDownloaded()) return SUCCESS_FROM_CACHE; QueueUrl = new URL( "http://api.netflix.com/users/" + user.getUserId() + "/queues/instant" + expanders); myQueueHandler = new InstantQueueHandler(); break; case NetFlixQueue.QUEUE_TYPE_DISC: if (!NetFlix.discQueue.isEmpty() && discQueue.isDownloaded()) return SUCCESS_FROM_CACHE; QueueUrl = new URL( "http://api.netflix.com/users/" + user.getUserId() + "/queues/disc/available" + expanders); myQueueHandler = new DiscQueueHandler(); break; } //Log.d("NetFlix", "" + QueueUrl.toString()); setSignPost(user.getAccessToken(), user.getAccessTokenSecret()); HttpURLConnection request = (HttpURLConnection) QueueUrl.openConnection(); Log.d("NetFlix", "getQueue() | ready"); NetFlix.oaconsumer.sign(request); Log.d("NetFlix", "getQueue() | signed"); request.connect(); Log.d("NetFlix", "getQueue() | response"); lastResponseMessage = request.getResponseCode() + ": " + request.getResponseMessage(); result = request.getResponseCode(); xml = request.getInputStream(); /* BufferedReader in = new BufferedReader(new InputStreamReader(xml)); String linein = null; while ((linein = in.readLine()) != null) { Log.d("NetFlix", "GetQueue: " + linein); }*/ SAXParserFactory spf = SAXParserFactory.newInstance(); SAXParser sp; sp = spf.newSAXParser(); XMLReader xr = sp.getXMLReader(); xr.setContentHandler(myQueueHandler); Log.d("NetFlix", "getQueue() | parse ready"); xr.parse(new InputSource(xml)); Log.d("NetFlix", "getQueue() | parse complete"); result = myQueueHandler.getSubCode(result); if (myQueueHandler.getMessage() != null) { //we may have an error from netflix, check it lastResponseMessage += " NF:" + result + ", " + myQueueHandler.getMessage(); lastNFResponseMessage = myQueueHandler.getMessage(); } else { lastNFResponseMessage = "No Message"; } if (result == 200) { switch (queueType) { case NetFlixQueue.QUEUE_TYPE_INSTANT: instantQueue.setDownloaded(true); break; case NetFlixQueue.QUEUE_TYPE_DISC: discQueue.setDownloaded(true); break; } } else if (result == 502) { HashMap<String, String> parameters = new HashMap<String, String>(); parameters.put("Queue Type:", "" + NetFlixQueue.queueTypeText[queueType]); parameters.put("HTTP Result:", "" + lastResponseMessage); parameters.put("User ID:", "" + user.getUserId()); parameters.put("NF Message:", "" + myQueueHandler.getMessage()); FlurryAgent.onEvent("getQueue502", parameters); } } catch (ParserConfigurationException e) { reportError(e, lastResponseMessage); } catch (SAXException e) { reportError(e, lastResponseMessage); } catch (IOException e) { reportError(e, lastResponseMessage); // Log.i("NetFlix", "IO Error connecting to NetFlix queue") } catch (OAuthMessageSignerException e) { reportError(e, lastResponseMessage); // Log.i("NetFlix", "Unable to Sign request - token invalid") } catch (OAuthExpectationFailedException e) { reportError(e, lastResponseMessage); // Log.i("NetFlix", "Expectation failed") } Log.d("NetFlix", "getQueue()<<<"); return result; }
From source file:org.apache.tez.engine.common.shuffle.impl.Fetcher.java
/** * The crux of the matter.../* w w w.j a v a 2s .c o m*/ * * @param host {@link MapHost} from which we need to * shuffle available map-outputs. */ @VisibleForTesting protected void copyFromHost(MapHost host) throws IOException { // Get completed maps on 'host' List<TezTaskAttemptID> maps = scheduler.getMapsForHost(host); // Sanity check to catch hosts with only 'OBSOLETE' maps, // especially at the tail of large jobs if (maps.size() == 0) { return; } if (LOG.isDebugEnabled()) { LOG.debug("Fetcher " + id + " going to fetch from " + host + " for: " + maps); } // List of maps to be fetched yet Set<TezTaskAttemptID> remaining = new HashSet<TezTaskAttemptID>(maps); // Construct the url and connect DataInputStream input; boolean connectSucceeded = false; try { URL url = getMapOutputURL(host, maps); HttpURLConnection connection = openConnection(url); // generate hash of the url String msgToEncode = SecureShuffleUtils.buildMsgFrom(url); String encHash = SecureShuffleUtils.hashFromString(msgToEncode, jobTokenSecret); // put url hash into http header connection.addRequestProperty(SecureShuffleUtils.HTTP_HEADER_URL_HASH, encHash); // set the read timeout connection.setReadTimeout(readTimeout); // put shuffle version into http header connection.addRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); connection.addRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION, ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); connect(connection, connectionTimeout); connectSucceeded = true; input = new DataInputStream(connection.getInputStream()); // Validate response code int rc = connection.getResponseCode(); if (rc != HttpURLConnection.HTTP_OK) { throw new IOException("Got invalid response code " + rc + " from " + url + ": " + connection.getResponseMessage()); } // get the shuffle version if (!ShuffleHeader.DEFAULT_HTTP_HEADER_NAME .equals(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_NAME)) || !ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION .equals(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_VERSION))) { throw new IOException("Incompatible shuffle response version"); } // get the replyHash which is HMac of the encHash we sent to the server String replyHash = connection.getHeaderField(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH); if (replyHash == null) { throw new IOException("security validation of TT Map output failed"); } LOG.debug("url=" + msgToEncode + ";encHash=" + encHash + ";replyHash=" + replyHash); // verify that replyHash is HMac of encHash SecureShuffleUtils.verifyReply(replyHash, encHash, jobTokenSecret); LOG.info("for url=" + msgToEncode + " sent hash and receievd reply"); } catch (IOException ie) { ioErrs.increment(1); LOG.warn("Failed to connect to " + host + " with " + remaining.size() + " map outputs", ie); // If connect did not succeed, just mark all the maps as failed, // indirectly penalizing the host if (!connectSucceeded) { for (TezTaskAttemptID left : remaining) { scheduler.copyFailed(left, host, connectSucceeded); } } else { // If we got a read error at this stage, it implies there was a problem // with the first map, typically lost map. So, penalize only that map // and add the rest TezTaskAttemptID firstMap = maps.get(0); scheduler.copyFailed(firstMap, host, connectSucceeded); } // Add back all the remaining maps, WITHOUT marking them as failed for (TezTaskAttemptID left : remaining) { scheduler.putBackKnownMapOutput(host, left); } return; } try { // Loop through available map-outputs and fetch them // On any error, faildTasks is not null and we exit // after putting back the remaining maps to the // yet_to_be_fetched list and marking the failed tasks. TezTaskAttemptID[] failedTasks = null; while (!remaining.isEmpty() && failedTasks == null) { failedTasks = copyMapOutput(host, input, remaining); } if (failedTasks != null && failedTasks.length > 0) { LOG.warn("copyMapOutput failed for tasks " + Arrays.toString(failedTasks)); for (TezTaskAttemptID left : failedTasks) { scheduler.copyFailed(left, host, true); } } IOUtils.cleanup(LOG, input); // Sanity check if (failedTasks == null && !remaining.isEmpty()) { throw new IOException( "server didn't return all expected map outputs: " + remaining.size() + " left."); } } finally { for (TezTaskAttemptID left : remaining) { scheduler.putBackKnownMapOutput(host, left); } } }
From source file:eionet.cr.harvest.PullHarvest.java
/** * Harvests external source./*w w w . j a v a2 s.c o m*/ * * @throws HarvestException * if harvest fails */ private void doUrlHarvest() throws HarvestException { String initialContextUrl = getContextUrl(); HttpURLConnection urlConn = null; httpResponseCode = NO_RESPONSE; String responseMessage = null; int noOfRedirections = 0; try { String connectUrl = getContextUrl(); do { String message = "Opening URL connection"; if (!connectUrl.equals(getContextUrl())) { message = message + " to " + connectUrl; } LOGGER.debug(loggerMsg(message)); urlConn = openUrlConnection(connectUrl); try { httpResponseCode = urlConn.getResponseCode(); responseMessage = urlConn.getResponseMessage(); } catch (IOException ioe) { // an error when connecting to server is considered a temporary error- // don't throw it, but log in the database and exit LOGGER.debug("Error when connecting to server: " + ioe); finishWithError(NO_RESPONSE, null, ioe); return; } // Throws exception when the content-length indicated in HTTP response is more than the maximum allowed. validateContentLength(urlConn); // Handle redirection. if (isRedirect(httpResponseCode)) { noOfRedirections++; // if number of redirections more than maximum allowed, throw exception if (noOfRedirections > MAX_REDIRECTIONS) { throw new TooManyRedirectionsException( "Too many redirections, originally started from " + initialContextUrl); } // get redirected-to-url, throw exception if it's missing String redirectedToUrl = getRedirectUrl(urlConn); redirectedUrls.add(connectUrl); redirectedHarvestSources.add(getContextSourceDTO()); if (StringUtils.isBlank(redirectedToUrl)) { throw new NoRedirectLocationException( "Redirection response code wihtout \"Location\" header!"); } LOGGER.debug(loggerMsg(connectUrl + " redirects to " + redirectedToUrl)); // treat this as a redirection only if the context URL and the redirected-to-URL // are not essentially the same if (!URLUtil.equalUrls(getContextUrl(), redirectedToUrl)) { finishRedirectedHarvest(redirectedToUrl, httpResponseCode); LOGGER.debug(loggerMsg("Redirection details saved")); startWithNewContext(redirectedToUrl); } else { LOGGER.debug(loggerMsg("Ignoring this redirection, as it is essentially to the same URL")); } connectUrl = redirectedToUrl; // Close redirected URL connection URLUtil.disconnect(urlConn); } } while (isRedirect(httpResponseCode)); // if URL connection returned no errors and its content has been modified since last harvest, // proceed to downloading if (!isError(httpResponseCode) && !isNotModified(httpResponseCode)) { int noOfTriples = downloadAndProcessContent(urlConn); setStoredTriplesCount(noOfTriples); LOGGER.debug(loggerMsg(noOfTriples + " triples loaded")); finishWithOK(urlConn, noOfTriples); } else if (isNotModified(httpResponseCode)) { LOGGER.debug(loggerMsg("Source not modified since last harvest")); finishWithNotModified(urlConn, 0); } else if (isError(httpResponseCode)) { LOGGER.debug(loggerMsg("Server returned error code " + httpResponseCode)); finishWithError(httpResponseCode, responseMessage, null); } } catch (Exception e) { LOGGER.debug(loggerMsg("Exception occurred (will be further logged by caller below): " + e.toString())); // check what caused the DAOException - fatal flag is set to true checkAndSetFatalExceptionFlag(e.getCause()); try { finishWithError(httpResponseCode, responseMessage, e); } catch (RuntimeException finishingException) { LOGGER.error("Error when finishing up: ", finishingException); } if (e instanceof HarvestException) { throw (HarvestException) e; } else { throw new HarvestException(e.getMessage(), e); } } finally { URLUtil.disconnect(urlConn); } }