List of usage examples for org.apache.commons.httpclient HttpStatus SC_NOT_MODIFIED
int SC_NOT_MODIFIED
To view the source code for org.apache.commons.httpclient HttpStatus SC_NOT_MODIFIED.
Click Source Link
From source file:fr.aliasource.webmail.server.proxy.client.http.AbstractClientMethod.java
protected InputStream executeStream(Map<String, String> parameters) { InputStream is = null;//from w w w . j a va 2 s . c o m PostMethod pm = new PostMethod(url); pm.setRequestHeader("Content-Type", "application/x-www-form-urlencoded; charset=utf-8"); for (String p : parameters.keySet()) { pm.setParameter(p, parameters.get(p)); } int ret = 0; synchronized (hc) { try { ret = hc.executeMethod(pm); if (ret == HttpStatus.SC_NOT_MODIFIED) { logger.info("backend wants us to use cached data"); } else if (ret != HttpStatus.SC_OK) { logger.error("method failed:\n" + pm.getStatusLine() + "\n" + pm.getResponseBodyAsString()); } else { is = pm.getResponseBodyAsStream(); ByteArrayOutputStream out = new ByteArrayOutputStream(); transfer(is, out, true); return new ByteArrayInputStream(out.toByteArray()); } } catch (Throwable e) { logger.error(e.getMessage(), e); } finally { pm.releaseConnection(); } } return is; }
From source file:com.discursive.jccook.httpclient.ConditionalGetExample.java
private void processResults(HttpMethod method) throws HttpException { if (method.getStatusCode() == HttpStatus.SC_NOT_MODIFIED) { System.out.println("Content not modified since last request"); } else {/*from w ww. j a v a 2 s .c om*/ System.out.println("Get Method retrieved content."); entityTag = retrieveHeader(method, "ETag"); lastModified = retrieveHeader(method, "Last-Modified"); System.out.println("Entity Tag: " + entityTag); System.out.println("Last Modified: " + lastModified); } }
From source file:com.sdm.core.resource.DefaultResource.java
protected DefaultResponse<MessageModel> validateCache(int cacheAge) { ResponseBuilder builder = request.evaluatePreconditions(lastModified); if (builder != null) { MessageModel message = new MessageModel(HttpStatus.SC_NOT_MODIFIED, "NO_CHANGE", "There is no any changes for your request."); DefaultResponse<MessageModel> response = new DefaultResponse<>(HttpStatus.SC_NOT_MODIFIED, ResponseType.INFO, message); response.setHeaders(this.buildCache(cacheAge)); return response; }/*from ww w . j a v a 2s .co m*/ return null; }
From source file:com.owncloud.android.lib.test_project.test.GetUserAvatarTest.java
/** * Test get user avatar only if changed, but wasn't changed *//*from w w w . j ava2 s .c om*/ public void testGetUserAvatarOnlyIfChangedAfterUnchanged() { RemoteOperationResult result = mActivity.getUserAvatar(AVATAR_DIMENSION, null); ResultData userAvatar = (ResultData) result.getData().get(0); String etag = userAvatar.getEtag(); // request again, with the just received etag result = mActivity.getUserAvatar(AVATAR_DIMENSION, etag); assertFalse(result.isSuccess()); assertTrue(result.getHttpCode() == HttpStatus.SC_NOT_MODIFIED); }
From source file:com.owncloud.android.lib.resources.activities.GetRemoteActivitiesOperation.java
private boolean isSuccess(int status) { return (status == HttpStatus.SC_OK || status == HttpStatus.SC_NOT_MODIFIED); }
From source file:com.cyberway.issue.crawler.writer.WARCWriterProcessor.java
protected void write(final String lowerCaseScheme, final CrawlURI curi) throws IOException { WriterPoolMember writer = getPool().borrowFile(); long position = writer.getPosition(); // See if we need to open a new file because we've exceeed maxBytes. // Call to checkFileSize will open new file if we're at maximum for // current file. writer.checkSize();/*from w ww .ja v a2 s . com*/ if (writer.getPosition() != position) { // We just closed the file because it was larger than maxBytes. // Add to the totalBytesWritten the size of the first record // in the file, if any. setTotalBytesWritten(getTotalBytesWritten() + (writer.getPosition() - position)); position = writer.getPosition(); } WARCWriter w = (WARCWriter) writer; try { // Write a request, response, and metadata all in the one // 'transaction'. final URI baseid = getRecordID(); final String timestamp = ArchiveUtils.getLog14Date(curi.getLong(A_FETCH_BEGAN_TIME)); if (lowerCaseScheme.startsWith("http")) { // Add named fields for ip, checksum, and relate the metadata // and request to the resource field. // TODO: Use other than ANVL (or rename ANVL as NameValue or // use RFC822 (commons-httpclient?). ANVLRecord headers = new ANVLRecord(5); if (curi.getContentDigest() != null) { headers.addLabelValue(HEADER_KEY_PAYLOAD_DIGEST, curi.getContentDigestSchemeString()); } headers.addLabelValue(HEADER_KEY_IP, getHostAddress(curi)); URI rid; if (IdenticalDigestDecideRule.hasIdenticalDigest(curi) && ((Boolean) getUncheckedAttribute(curi, ATTR_WRITE_REVISIT_FOR_IDENTICAL_DIGESTS))) { rid = writeRevisitDigest(w, timestamp, HTTP_RESPONSE_MIMETYPE, baseid, curi, headers); } else if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED && ((Boolean) getUncheckedAttribute(curi, ATTR_WRITE_REVISIT_FOR_NOT_MODIFIED))) { rid = writeRevisitNotModified(w, timestamp, baseid, curi, headers); } else { if (curi.isTruncatedFetch()) { String value = curi.isTimeTruncatedFetch() ? NAMED_FIELD_TRUNCATED_VALUE_TIME : curi.isLengthTruncatedFetch() ? NAMED_FIELD_TRUNCATED_VALUE_LENGTH : curi.isHeaderTruncatedFetch() ? NAMED_FIELD_TRUNCATED_VALUE_HEAD : // TODO: Add this to spec. TRUNCATED_VALUE_UNSPECIFIED; headers.addLabelValue(HEADER_KEY_TRUNCATED, value); } rid = writeResponse(w, timestamp, HTTP_RESPONSE_MIMETYPE, baseid, curi, headers); } headers = new ANVLRecord(1); headers.addLabelValue(HEADER_KEY_CONCURRENT_TO, '<' + rid.toString() + '>'); if (((Boolean) getUncheckedAttribute(curi, ATTR_WRITE_REQUESTS))) { writeRequest(w, timestamp, HTTP_REQUEST_MIMETYPE, baseid, curi, headers); } if (((Boolean) getUncheckedAttribute(curi, ATTR_WRITE_METADATA))) { writeMetadata(w, timestamp, baseid, curi, headers); } } else if (lowerCaseScheme.equals("dns")) { ANVLRecord headers = null; String ip = curi.getString(A_DNS_SERVER_IP_LABEL); if (ip != null && ip.length() > 0) { headers = new ANVLRecord(1); headers.addLabelValue(HEADER_KEY_IP, ip); } writeResponse(w, timestamp, curi.getContentType(), baseid, curi, headers); } else { logger.warning("No handler for scheme " + lowerCaseScheme); } } catch (IOException e) { // Invalidate this file (It gets a '.invalid' suffix). getPool().invalidateFile(writer); // Set the writer to null otherwise the pool accounting // of how many active writers gets skewed if we subsequently // do a returnWriter call on this object in the finally block. writer = null; throw e; } finally { if (writer != null) { setTotalBytesWritten(getTotalBytesWritten() + (writer.getPosition() - position)); getPool().returnFile(writer); } } checkBytesWritten(); }
From source file:com.dtolabs.rundeck.core.common.impl.URLFileUpdater.java
private void updateHTTPUrl(final File destinationFile) throws FileUpdaterException { if (null == interaction) { interaction = new normalInteraction(); }//from w w w . j a v a2 s. co m final Properties cacheProperties; if (useCaching) { cacheProperties = loadCacheData(cacheMetadata); contentTypeFromCache(cacheProperties); } else { cacheProperties = null; } final HttpClientParams params = new HttpClientParams(); if (timeout > 0) { params.setConnectionManagerTimeout(timeout * 1000); params.setSoTimeout(timeout * 1000); } final HttpClient client = new HttpClient(params); AuthScope authscope = null; UsernamePasswordCredentials cred = null; boolean doauth = false; String cleanUrl = url.toExternalForm().replaceAll("^(https?://)([^:@/]+):[^@/]*@", "$1$2:****@"); String urlToUse = url.toExternalForm(); try { if (null != url.getUserInfo()) { doauth = true; authscope = new AuthScope(url.getHost(), url.getPort() > 0 ? url.getPort() : url.getDefaultPort(), AuthScope.ANY_REALM, "BASIC"); cred = new UsernamePasswordCredentials(url.getUserInfo()); urlToUse = new URL(url.getProtocol(), url.getHost(), url.getPort(), url.getFile()).toExternalForm(); } else if (null != username && null != password) { doauth = true; authscope = new AuthScope(url.getHost(), url.getPort() > 0 ? url.getPort() : url.getDefaultPort(), AuthScope.ANY_REALM, "BASIC"); cred = new UsernamePasswordCredentials(username + ":" + password); urlToUse = new URL(url.getProtocol(), url.getHost(), url.getPort(), url.getFile()).toExternalForm(); } } catch (MalformedURLException e) { throw new FileUpdaterException("Failed to configure base URL for authentication: " + e.getMessage(), e); } if (doauth) { client.getParams().setAuthenticationPreemptive(true); client.getState().setCredentials(authscope, cred); } interaction.setClient(client); interaction.setMethod(new GetMethod(urlToUse)); interaction.setFollowRedirects(true); if (null != acceptHeader) { interaction.setRequestHeader("Accept", acceptHeader); } else { interaction.setRequestHeader("Accept", "*/*"); } if (useCaching) { applyCacheHeaders(cacheProperties, interaction); } logger.debug("Making remote request: " + cleanUrl); try { resultCode = interaction.executeMethod(); reasonCode = interaction.getStatusText(); if (useCaching && HttpStatus.SC_NOT_MODIFIED == resultCode) { logger.debug("Content NOT MODIFIED: file up to date"); } else if (HttpStatus.SC_OK == resultCode) { determineContentType(interaction); //write to file FileOutputStream output = new FileOutputStream(destinationFile); try { Streams.copyStream(interaction.getResponseBodyAsStream(), output); } finally { output.close(); } if (destinationFile.length() < 1) { //file was empty! if (!destinationFile.delete()) { logger.warn("Failed to remove empty file: " + destinationFile.getAbsolutePath()); } } if (useCaching) { cacheResponseInfo(interaction, cacheMetadata); } } else { throw new FileUpdaterException( "Unable to retrieve content: result code: " + resultCode + " " + reasonCode); } } catch (HttpException e) { throw new FileUpdaterException(e); } catch (IOException e) { throw new FileUpdaterException(e); } finally { interaction.releaseConnection(); } }
From source file:dk.netarkivet.harvester.harvesting.WARCWriterProcessor.java
private void writeHttpRecords(WARCWriter w, final CrawlURI curi, final URI baseid, final String timestamp) throws IOException { // Add named fields for ip, checksum, and relate the metadata // and request to the resource field. // TODO: Use other than ANVL (or rename ANVL as NameValue or // use RFC822 (commons-httpclient?). ANVLRecord headers = new ANVLRecord(5); if (curi.getContentDigest() != null) { headers.addLabelValue(HEADER_KEY_PAYLOAD_DIGEST, curi.getContentDigestSchemeString()); }//from w w w . j av a2 s.c o m headers.addLabelValue(HEADER_KEY_IP, getHostAddress(curi)); URI rid; if (IdenticalDigestDecideRule.hasIdenticalDigest(curi) && ((Boolean) getUncheckedAttribute(curi, ATTR_WRITE_REVISIT_FOR_IDENTICAL_DIGESTS))) { rid = writeRevisitDigest(w, timestamp, HTTP_RESPONSE_MIMETYPE, baseid, curi, headers); } else if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED && ((Boolean) getUncheckedAttribute(curi, ATTR_WRITE_REVISIT_FOR_NOT_MODIFIED))) { rid = writeRevisitNotModified(w, timestamp, baseid, curi, headers); } else { if (curi.isTruncatedFetch()) { String value = curi.isTimeTruncatedFetch() ? NAMED_FIELD_TRUNCATED_VALUE_TIME : curi.isLengthTruncatedFetch() ? NAMED_FIELD_TRUNCATED_VALUE_LENGTH : curi.isHeaderTruncatedFetch() ? NAMED_FIELD_TRUNCATED_VALUE_HEAD : // TODO: Add this to spec. TRUNCATED_VALUE_UNSPECIFIED; headers.addLabelValue(HEADER_KEY_TRUNCATED, value); } rid = writeResponse(w, timestamp, HTTP_RESPONSE_MIMETYPE, baseid, curi, headers); } headers = new ANVLRecord(1); headers.addLabelValue(HEADER_KEY_CONCURRENT_TO, '<' + rid.toString() + '>'); if (((Boolean) getUncheckedAttribute(curi, ATTR_WRITE_REQUESTS))) { writeRequest(w, timestamp, HTTP_REQUEST_MIMETYPE, baseid, curi, headers); } if (((Boolean) getUncheckedAttribute(curi, ATTR_WRITE_METADATA))) { writeMetadata(w, timestamp, baseid, curi, headers); } }
From source file:com.cyberway.issue.crawler.fetcher.FetchHTTP.java
/** * Update CrawlURI internal sizes based on current transaction (and * in the case of 304s, history) // ww w . ja va 2 s .c o m * * @param curi CrawlURI * @param rec HttpRecorder */ protected void setSizes(final CrawlURI curi, HttpRecorder rec) { // set reporting size curi.setContentSize(rec.getRecordedInput().getSize()); // special handling for 304-not modified if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED && curi.containsKey(A_FETCH_HISTORY)) { AList history[] = curi.getAList().getAListArray(A_FETCH_HISTORY); if (history[0] != null && history[0].containsKey(CoreAttributeConstants.A_REFERENCE_LENGTH)) { long referenceLength = history[0].getLong(A_REFERENCE_LENGTH); // carry-forward previous 'reference-length' for future curi.putLong(A_REFERENCE_LENGTH, referenceLength); // increase content-size to virtual-size for reporting curi.setContentSize(rec.getRecordedInput().getSize() + referenceLength); } } }
From source file:com.silverwrist.venice.std.TrackbackManager.java
/** * Loads the HTTP content at the specified URL, scans it for RDF description blocks, and adds those blocks * as {@link com.silverwrist.venice.std.TrackbackItem TrackbackItem}s to our internal cache. Uses modification * detection to keep from reloading a page unless necessary. * * @param url The URL of the resource to be loaded. * @param attrs The attributes of the specified page; if this is <code>null</code>, we'll check the page * cache for the right attributes. * @return <code>true</code> if the page data was loaded and scanned for trackback items; <code>false</code> * if no data was loaded (because it was not modified since the last time we loaded it, for instance). * @exception com.silverwrist.venice.except.TrackbackException If there was an error loading or interpreting * the page data./*from w w w .j a v a 2 s .c o m*/ */ private synchronized boolean load(URL url, PageAttributes attrs) throws TrackbackException { if (attrs == null) attrs = (PageAttributes) (m_page_cache.get(url)); // Create the GET method and set its headers. String s = url.toString(); int x = s.lastIndexOf('#'); if (x >= 0) s = s.substring(0, x); GetMethod getter = new GetMethod(s); HttpMethodParams params = getter.getParams(); getter.setDoAuthentication(false); getter.setFollowRedirects(true); getter.setRequestHeader("User-Agent", USER_AGENT); getter.setRequestHeader("Accept", "text/*"); getter.setRequestHeader("Accept-Encoding", "identity"); params.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler()); getter.setParams(params); boolean get_resp = false; PageAttributes newattrs = null; ContentType ctype = null; byte[] rawdata = null; try { // set the Last-Modified date as an If-Modified-Since header on the request java.util.Date lmod = null; if (attrs != null) lmod = attrs.getLastModified(); if (lmod != null) getter.setRequestHeader("If-Modified-Since", s_httpdate_format.format(lmod)); // execute the Get method! int rc = m_http_client.executeMethod(getter); get_resp = true; if ((lmod != null) && (rc == HttpStatus.SC_NOT_MODIFIED)) return false; // we were not modified if (rc == HttpStatus.SC_NO_CONTENT) return false; // there's no content there if (rc != HttpStatus.SC_OK) // this is farked! throw new TrackbackException("GET of " + url + " returned " + rc); // Get the new page attributes and save them off. newattrs = new PageAttributes(getter); m_page_cache.put(url, newattrs); // Get the Content-Type header and see if it's valid. Header hdr = getter.getResponseHeader("Content-Type"); if (hdr != null) s = hdr.getValue(); else s = "text/plain"; // necessary assumption ctype = new ContentType(s); if (!(ctype.getPrimaryType().equals("text"))) throw new TrackbackException("URL " + url + " does not point to a text-based resource"); // Load the resource in as byte data; we will determine the right character set for it later. rawdata = getter.getResponseBody(); get_resp = false; } // end try catch (IOException e) { // IO error getting the page throw new TrackbackException("I/O error retrieving " + url + ": " + e.getMessage(), e); } // end catch catch (javax.mail.internet.ParseException e) { // translate into TrackbackException throw new TrackbackException("invalid Content-Type received for URL " + url, e); } // end catch finally { // release the connection if possible try { // need to get the message body if (get_resp) getter.getResponseBody(); } // end try catch (IOException e) { // ignore these } // end catch getter.releaseConnection(); } // end finally // make a first guess at the charset from the HTTP header Content-Type String cset = ctype.getParameter("charset"); if (cset == null) cset = "US-ASCII"; String content = null; try { // interpret the content content = new String(rawdata, cset); } // end try catch (UnsupportedEncodingException e) { // fall back and try just using US-ASCII cset = null; try { // interpret the content content = new String(rawdata, "US-ASCII"); } // end try catch (UnsupportedEncodingException e2) { // can't happen logger.debug("WTF? US-ASCII should damn well be a supported character set!", e2); } // end catch } // end catch // Look for <META HTTP-EQUIV=...> tags in the content. Map http_attrs = extractHttpEquivTags(content); // Try to get a Content-Type attribute from there. s = (String) (http_attrs.get("CONTENT-TYPE")); String cset2 = null; if (s != null) { // look for the content type try { // parse into Content-Type ContentType c = new ContentType(s); if (c.getPrimaryType().equals("text")) cset2 = c.getParameter("charset"); } // end try catch (javax.mail.internet.ParseException e) { // can't get a second Content-Type logger.debug("parse of Content-Type from META tags failed", e); cset2 = null; } // end catch } // end if if ((cset == null) && (cset2 == null)) throw new TrackbackException("unable to determine character set for " + url); if ((cset2 != null) && ((cset == null) || !(cset.equalsIgnoreCase(cset2)))) { // reinterpret content in new character set try { // reinterpret content in new character set s = new String(rawdata, cset2); content = s; // the contents of the HTTP-EQUIV tags may have changed as a result http_attrs = extractHttpEquivTags(content); } // end try catch (UnsupportedEncodingException e) { // just use original character set if (cset == null) throw new TrackbackException("unable to determine character set for " + url); } // end catch } // end if newattrs.updateFromPage(http_attrs); // update the page attributes from the META tag data // Search the page content for RDF blocks. RE m = new RE(s_rdf_start, RE.MATCH_NORMAL); int pos = 0; while (m.match(content, pos)) { // look for the end of this RDF block RE m2 = new RE(getEndRecognizer(m.getParen(1)), RE.MATCH_NORMAL); if (m2.match(content, m.getParenEnd(0))) { // we now have a block to feed to the XML parser try { // run the block through the XML parser InputSource isrc = new InputSource( new StringReader(content.substring(m.getParenStart(0), m2.getParenEnd(0)))); Document doc = m_rdf_parser.parse(isrc); // examine topmost element, which should be rdf:RDF Element root = doc.getDocumentElement(); if (NS_RDF.equals(root.getNamespaceURI()) && (root.getLocalName() != null) && root.getLocalName().equals("RDF")) { // this is most definitely an rdf:RDF node...look for rdf:Description nodes under it NodeList nl = root.getChildNodes(); for (int i = 0; i < nl.getLength(); i++) { // check each node in the list Node n = nl.item(i); if ((n.getNodeType() == Node.ELEMENT_NODE) && NS_RDF.equals(n.getNamespaceURI()) && (n.getLocalName() != null) && n.getLocalName().equals("Description")) { // we've got an rdf:Description node...extract the attributes from it Element elt = (Element) n; try { // look for the item and trackback URLs URL item = null, trackback = null; s = elt.getAttributeNS(NS_DC, "identifier"); if ((s != null) && (s.length() > 0)) item = new URL(s); s = elt.getAttributeNS(NS_TRACKBACK, "ping"); if ((s != null) && (s.length() > 0)) trackback = new URL(s); if ((item != null) && (trackback != null)) { // create the item s = elt.getAttributeNS(NS_DC, "title"); m_item_cache.put(item, new MyTrackbackItem(item, trackback, s, newattrs)); } // end if } // end try catch (MalformedURLException e) { // this means skip this item logger.warn("URL parse failure", e); } // end catch } // end if } // end for } // end if } // end try catch (IOException e) { // disregard this block logger.warn("RDF block parse failure", e); } // end catch catch (SAXException e) { // disregard this block logger.warn("RDF block parse failure", e); } // end catch } // end if // else ignore this possible block pos = m.getParenEnd(0); } // end while return true; }