List of usage examples for java.net HttpURLConnection getLastModified
public long getLastModified()
From source file:com.centurylink.mdw.ant.taskdef.HttpTransfer.java
public void uploadFile(URL destURL, File file, String userId, String password, boolean overwrite) { try {/*w w w.ja v a 2 s . c o m*/ log("target url... " + destURL.toString()); long fileLastModified = file.lastModified(); HttpURLConnection conn = (HttpURLConnection) destURL.openConnection(); long urlLastModified = conn.getLastModified(); conn.disconnect(); if (!overwrite && (urlLastModified >= fileLastModified)) { log("Destination file is up-to-date, not uploading."); return; } else { conn = (HttpURLConnection) destURL.openConnection(); conn.setRequestProperty("Content-Type", "application/octet-stream"); conn.setRequestMethod("PUT"); if (userId != null) { String value = userId + ":" + password; conn.setRequestProperty("Authorization", "Basic " + new String(Base64.encodeBase64(value.getBytes()))); } conn.setDoOutput(true); OutputStream outStream = conn.getOutputStream(); log("Uploading... " + file); InputStream inStream = new FileInputStream(file); byte[] buf = new byte[1024]; int len = 0; while (len != -1) { len = inStream.read(buf); if (len > 0) outStream.write(buf, 0, len); } inStream.close(); outStream.close(); conn.disconnect(); int code = conn.getResponseCode(); if (code < 200 || code >= 300) { String response = conn.getResponseMessage(); throw new BuildException("Error uploading file: " + code + " -- " + response); } log(" Uploaded: " + destURL); } } catch (IOException e) { if (isFailOnError()) throw new BuildException(e.getMessage(), e); else log(e.getMessage()); } }
From source file:biz.gabrys.lesscss.extended.compiler.source.HttpSource.java
public String getContent() { final HttpURLConnection connection = makeConnection(true); encoding = getEncodingFromContentType(connection.getContentType()); String content;/*from w w w. j a va 2 s . com*/ try { content = IOUtils.toString(connection.getInputStream(), encoding); } catch (final IOException e) { connection.disconnect(); throw new SourceException(e); } lastModificationDate = getModificationDate(connection.getLastModified()); connection.disconnect(); return content; }
From source file:org.transitime.utils.HttpGetFile.java
/** * Actually gets and stores the file. The User-Agency property is always set * to USER_AGENT./* www . j av a2s. c o m*/ * * @return The http response code such as HttpStatus.SC_OK * @throws IOException */ public int getFile() throws IOException { IntervalTimer timer = new IntervalTimer(); logger.debug("Getting URL={}", urlStr); URL url = new URL(urlStr); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); connection.setRequestProperty("User-Agency", USER_AGENT); // Set request properties for (int i = 0; i < headerKeys.size(); ++i) { connection.setRequestProperty(headerKeys.get(i), headerValues.get(i)); } // Get and log response code int responseCode = connection.getResponseCode(); long expectedContentLength = connection.getContentLengthLong(); long remoteFileLastModified = connection.getLastModified(); logger.debug( "Response code for getting file {} is {} and file size " + "is {} bytes and remote file lastModified=\"{}\" or {} msec", urlStr, responseCode, expectedContentLength, Time.httpDate(remoteFileLastModified), remoteFileLastModified); // Open file for where results are to be written File file = new File(fullFileNameForResult); // If file could not be read in or is not newer that lastModified time // of the existing file on the server then don't need to continue // reading it in. if (responseCode != HttpStatus.SC_OK) { logger.debug("Response code was {} so not reading in file", responseCode); return responseCode; } // Sometimes a web server will return http status OK (200) even // when the remote file is older than the time set for If-Modified-Since // header. For this situation still don't want to read in the file // so simply return http status NO_MODIFIED (304). if (file.lastModified() > 0 && remoteFileLastModified < file.lastModified()) { logger.warn("Response code was {} but the local file was modified " + "after the remote file so it must be up to date. " + "Therefore remote file not read in.", responseCode); return HttpStatus.SC_NOT_MODIFIED; } logger.debug( "Actually reading data from URL {} . Local file " + "lastModified={} or {} msec and remoteFileLastModified={} " + "or {} msec.", urlStr, Time.httpDate(file.lastModified()), file.lastModified(), Time.httpDate(remoteFileLastModified), remoteFileLastModified); // Make sure output directory exists file.getParentFile().mkdirs(); // Open input stream for reading data InputStream in = connection.getInputStream(); // Open the stream FileOutputStream fos = new FileOutputStream(file); IntervalTimer loopTimer = new IntervalTimer(); long lengthSinceLoggingMsg = 0; // Copy contents to file byte[] buffer = new byte[4096]; int length; int totalLength = 0; while ((length = in.read(buffer)) > 0) { fos.write(buffer, 0, length); totalLength += length; lengthSinceLoggingMsg += length; // Every once in a while log progress. Don't want to // check timer every loop since that would be expensive. // So only check timer for every MB downloaded. if (lengthSinceLoggingMsg > 1024 * 1024) { lengthSinceLoggingMsg = 0; if (loopTimer.elapsedMsec() > 10 * Time.MS_PER_SEC) { loopTimer.resetTimer(); logger.debug("Read in {} bytes or {}% of file {}", totalLength, StringUtils.oneDigitFormat(100.0 * totalLength / expectedContentLength), urlStr); } } } // Close things up fos.close(); // Set the last modified time so that it is the same as on the // web server. file.setLastModified(connection.getLastModified()); if (totalLength == expectedContentLength) logger.debug("Successfully copied {} to file {}. Length was {} " + "bytes. Took {} msec.", urlStr, fullFileNameForResult, totalLength, timer.elapsedMsec()); else logger.error("When copying {} to file {} the expected length was " + "{} but only copied {} bytes", urlStr, fullFileNameForResult, expectedContentLength, totalLength); // Return the http response code such as 200 for OK or 304 for // Not Modified return connection.getResponseCode(); }
From source file:org.apache.slider.core.restclient.UrlConnectionOperations.java
public HttpOperationResponse execHttpOperation(HttpVerb verb, URL url, byte[] payload, String contentType) throws IOException, AuthenticationException { HttpURLConnection conn = null; HttpOperationResponse outcome = new HttpOperationResponse(); int resultCode; byte[] body = null; log.debug("{} {} spnego={}", verb, url, useSpnego); boolean doOutput = verb.hasUploadBody(); if (doOutput) { Preconditions.checkArgument(payload != null, "Null payload on a verb which expects one"); }//from w w w . ja v a 2 s . c o m try { conn = openConnection(url); conn.setRequestMethod(verb.getVerb()); conn.setDoOutput(doOutput); if (doOutput) { conn.setRequestProperty("Content-Type", contentType); } // now do the connection conn.connect(); if (doOutput) { OutputStream output = conn.getOutputStream(); IOUtils.write(payload, output); output.close(); } resultCode = conn.getResponseCode(); outcome.lastModified = conn.getLastModified(); outcome.contentType = conn.getContentType(); outcome.headers = conn.getHeaderFields(); InputStream stream = conn.getErrorStream(); if (stream == null) { stream = conn.getInputStream(); } if (stream != null) { // read into a buffer. body = IOUtils.toByteArray(stream); } else { // no body: log.debug("No body in response"); } } catch (SSLException e) { throw e; } catch (IOException e) { throw NetUtils.wrapException(url.toString(), url.getPort(), "localhost", 0, e); } catch (AuthenticationException e) { throw new AuthenticationException("From " + url + ": " + e, e); } finally { if (conn != null) { conn.disconnect(); } } uprateFaults(HttpVerb.GET, url.toString(), resultCode, "", body); outcome.responseCode = resultCode; outcome.data = body; return outcome; }
From source file:org.broad.igv.util.HttpUtils.java
public long getLastModified(URL url) throws IOException { HttpURLConnection conn = openConnectionHeadOrGet(url); if (conn == null) return 0; return conn.getLastModified(); }
From source file:alluxio.client.rest.S3ClientRestApiTest.java
@Test public void getObjectMetadata() throws Exception { final String bucket = "bucket"; createBucketRestCall(bucket);/* w w w . ja va2 s . co m*/ final String objectKey = bucket + AlluxioURI.SEPARATOR + "object.txt"; final byte[] objectContent = CommonUtils.randomAlphaNumString(10).getBytes(); createObjectRestCall(objectKey, objectContent, null, NO_PARAMS); HttpURLConnection connection = getObjectMetadataRestCall(objectKey); URIStatus status = mFileSystem.getStatus(new AlluxioURI(AlluxioURI.SEPARATOR + objectKey)); // remove the milliseconds from the last modification time because the accuracy of HTTP dates // is up to seconds. long lastModified = status.getLastModificationTimeMs() / 1000 * 1000; Assert.assertEquals(lastModified, connection.getLastModified()); Assert.assertEquals(String.valueOf(status.getLength()), connection.getHeaderField(S3Constants.S3_CONTENT_LENGTH_HEADER)); }
From source file:edu.umn.cs.spatialHadoop.nasa.HTTPFileSystem.java
/** * Returns the status of a file. This method is designed specifically to work * with LP DAAC archive and will not work correctly with other web sites. * Since HTTP does not tell whether a URL points to a file or directory, * we assume that URLs ending with HDF, XML and JPG are files while anything * else is considered a directory./*from w w w. jav a 2s.co m*/ */ @Override public FileStatus getFileStatus(Path f) throws IOException { f = f.makeQualified(this); URL url = f.toUri().toURL(); int retryCount = HTTPFileSystem.retries; HttpURLConnection connection = null; try { while (connection == null && retryCount-- > 0) { try { connection = (HttpURLConnection) url.openConnection(); } catch (java.net.SocketException e) { if (retryCount == 0) throw e; LOG.info("Error accessing file '" + url + "'. Trials left: " + retryCount); try { ; Thread.sleep(1000); } catch (InterruptedException e1) { } } catch (java.net.UnknownHostException e) { if (retryCount == 0) throw e; LOG.info("Error accessing file '" + url + "'. Trials left: " + retryCount); try { Thread.sleep(1000); } catch (InterruptedException e1) { } } } if (connection == null) throw new RuntimeException("Could not connect to " + f); String lengthStr = connection.getHeaderField("content-Length"); long length = lengthStr == null ? -1 : Long.parseLong(lengthStr); if (length == -1) LOG.info("Unknown HTTP file length " + length); long modificationTime = connection.getLastModified(); if (modificationTime == 0) modificationTime = connection.getDate(); // Hard coded to work with LP DAAC archives boolean isdir = !f.getName().matches("(?i:([^*\\?])*\\.(hdf|xml|jpg|gz|bz2|zip|txt|csv|tsv)$)"); return new FileStatus(length, isdir, 1, BLOCK_SIZE, modificationTime, 0, null, null, null, f); } finally { if (connection != null) connection.disconnect(); } }
From source file:com.wanikani.wklib.Connection.java
protected Response call(Meter meter, String resource, boolean isArray, String arg, CacheInfo cinfo) throws IOException { HttpURLConnection conn; JSONTokener tok;//from w w w .j a v a 2 s . c om InputStream is; URL url; url = new URL(makeURL(resource, arg)); conn = null; tok = null; try { conn = (HttpURLConnection) url.openConnection(); if (cinfo != null) { if (cinfo.etag != null) conn.setRequestProperty("If-None-Match", cinfo.etag); else if (cinfo.modified != null) conn.setIfModifiedSince(cinfo.modified.getTime()); } setTimeouts(conn); conn.connect(); if (cinfo != null && cinfo.hasData() && conn.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) throw new NotModifiedException(); measureHeaders(meter, conn, false); is = conn.getInputStream(); tok = new JSONTokener(readStream(meter, is)); } finally { if (conn != null) conn.disconnect(); } if (cinfo != null) { cinfo.modified = new Date(); if (conn.getDate() > 0) cinfo.modified = new Date(conn.getDate()); if (conn.getLastModified() > 0) cinfo.modified = new Date(conn.getLastModified()); cinfo.etag = conn.getHeaderField("ETag"); } try { return new Response(new JSONObject(tok), isArray); } catch (JSONException e) { throw new ParseException(); } }
From source file:org.ocelotds.integration.AbstractOcelotTest.java
/** * Rcupere la resource via un HttpConnection * * @param resource//from ww w . j a va2 s .c o m * @return * @throws MalformedURLException * @throws IOException */ protected HttpURLConnection getConnectionForResource(String resource) throws MalformedURLException, IOException { StringBuilder sb = new StringBuilder("http://localhost:"); sb.append(PORT).append(Constants.SLASH).append(CTXPATH).append(Constants.SLASH).append(resource); URL url = new URL(sb.toString()); HttpURLConnection uc = (HttpURLConnection) url.openConnection(); // Authenticator.setDefault(new Authenticator() { // @Override // protected PasswordAuthentication getPasswordAuthentication() { // return new PasswordAuthentication("demo", "demo".toCharArray()); // } // }); // ou // String basicAuth = "Basic " + javax.xml.bind.DatatypeConverter.printBase64Binary("demo:demo".getBytes()); // System.out.println(basicAuth); // uc.setRequestProperty("Authorization", basicAuth); System.out.println("Content-type: " + uc.getContentType()); System.out.println("Date: " + new Date(uc.getDate())); System.out.println("Last modified: " + new Date(uc.getLastModified())); System.out.println("Expiration date: " + new Date(uc.getExpiration())); System.out.println("Response code: " + uc.getResponseCode()); assertThat(uc.getResponseCode()).isEqualTo(200).as("'%s' is unreachable", sb); return uc; }
From source file:org.apache.maven.wagon.providers.http.LightweightHttpWagon.java
public void fillInputData(InputData inputData) throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException { Resource resource = inputData.getResource(); String visitingUrl = buildUrl(resource.getName()); try {// w w w . j a v a2 s .c om List<String> visitedUrls = new ArrayList<String>(); for (int redirectCount = 0; redirectCount < MAX_REDIRECTS; redirectCount++) { if (visitedUrls.contains(visitingUrl)) { throw new TransferFailedException("Cyclic http redirect detected. Aborting! " + visitingUrl); } visitedUrls.add(visitingUrl); URL url = new URL(visitingUrl); HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(this.proxy); urlConnection.setRequestProperty("Accept-Encoding", "gzip"); if (!useCache) { urlConnection.setRequestProperty("Pragma", "no-cache"); } addHeaders(urlConnection); // TODO: handle all response codes int responseCode = urlConnection.getResponseCode(); if (responseCode == HttpURLConnection.HTTP_FORBIDDEN || responseCode == HttpURLConnection.HTTP_UNAUTHORIZED) { throw new AuthorizationException("Access denied to: " + buildUrl(resource.getName())); } if (responseCode == HttpURLConnection.HTTP_MOVED_PERM || responseCode == HttpURLConnection.HTTP_MOVED_TEMP) { visitingUrl = urlConnection.getHeaderField("Location"); continue; } InputStream is = urlConnection.getInputStream(); String contentEncoding = urlConnection.getHeaderField("Content-Encoding"); boolean isGZipped = contentEncoding != null && "gzip".equalsIgnoreCase(contentEncoding); if (isGZipped) { is = new GZIPInputStream(is); } inputData.setInputStream(is); resource.setLastModified(urlConnection.getLastModified()); resource.setContentLength(urlConnection.getContentLength()); break; } } catch (MalformedURLException e) { throw new ResourceDoesNotExistException("Invalid repository URL: " + e.getMessage(), e); } catch (FileNotFoundException e) { throw new ResourceDoesNotExistException("Unable to locate resource in repository", e); } catch (IOException e) { StringBuilder message = new StringBuilder("Error transferring file: "); message.append(e.getMessage()); message.append(" from " + visitingUrl); if (getProxyInfo() != null && getProxyInfo().getHost() != null) { message.append(" with proxyInfo ").append(getProxyInfo().toString()); } throw new TransferFailedException(message.toString(), e); } }