Java tutorial
/* $Id: DataCache.java 988245 2010-08-23 18:39:35Z kwright $ */ /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.manifoldcf.crawler.connectors.rss; import org.apache.manifoldcf.core.interfaces.*; import org.apache.manifoldcf.agents.interfaces.*; import org.apache.manifoldcf.crawler.interfaces.*; import org.apache.manifoldcf.crawler.system.Logging; import org.apache.manifoldcf.crawler.system.ManifoldCF; import java.util.*; import java.io.*; import org.apache.http.conn.ConnectTimeoutException; import org.apache.http.client.RedirectException; import org.apache.http.client.CircularRedirectException; import org.apache.http.NoHttpResponseException; import org.apache.http.HttpException; /** This class is a cache of a specific URL's data. It's fetched early and kept, * so that (1) an accurate data length can be found, and (2) we can compute a version * checksum. */ public class DataCache { public static final String _rcsid = "@(#)$Id: DataCache.java 988245 2010-08-23 18:39:35Z kwright $"; // Hashmap containing the cache protected Map<String, DocumentData> cacheData = new HashMap<String, DocumentData>(); /** Constructor. */ public DataCache() { } /** Add binary data entry into the cache. Does NOT close the input stream when done! *@param documentIdentifier is the document identifier (url). *@param contentType is the content type for the data. *@param dataStream is the data stream. *@return the checksum value. */ public long addData(IProcessActivity activities, String documentIdentifier, String contentType, InputStream dataStream) throws ManifoldCFException, ServiceInterruption { // Create a temporary file; that's what we will cache try { File tempFile = File.createTempFile("_rsscache_", "tmp"); try { // Causes memory leaks if left around; there's no way to release // the record specifying that the file should be deleted, even // after it's removed. So disable this and live with the occasional // dangling file left as a result of shutdown or error. :-( // tempFile.deleteOnExit(); ManifoldCF.addFile(tempFile); // Transfer data to temporary file long checkSum = 0L; OutputStream os = new FileOutputStream(tempFile); try { byte[] byteArray = new byte[65536]; while (true) { int amt; try { amt = dataStream.read(byteArray, 0, byteArray.length); } catch (java.net.SocketTimeoutException e) { Logging.connectors.warn( "RSS: Socket timeout exception reading socket stream: " + e.getMessage(), e); long currentTime = System.currentTimeMillis(); throw new ServiceInterruption("Read timeout: " + e.getMessage(), e, currentTime + 300000L, currentTime + 12 * 60 * 60000L, -1, false); } catch (ConnectTimeoutException e) { Logging.connectors.warn( "RSS: Connect timeout exception reading socket stream: " + e.getMessage(), e); long currentTime = System.currentTimeMillis(); throw new ServiceInterruption("Read timeout: " + e.getMessage(), e, currentTime + 300000L, currentTime + 12 * 60 * 60000L, -1, false); } catch (InterruptedIOException e) { throw new ManifoldCFException("Interrupted: " + e.getMessage(), ManifoldCFException.INTERRUPTED); } catch (IOException e) { Logging.connectors.warn("RSS: IO exception reading socket stream: " + e.getMessage(), e); long currentTime = System.currentTimeMillis(); throw new ServiceInterruption("Read timeout: " + e.getMessage(), e, currentTime + 300000L, currentTime + 12 * 60 * 60000L, -1, false); } if (amt == -1) break; int i = 0; while (i < amt) { byte x = byteArray[i++]; long bytevalue = (long) x; checkSum = (checkSum << 5) ^ (checkSum >> 3) ^ (bytevalue << 2) ^ (bytevalue >> 3); } os.write(byteArray, 0, amt); // Before we go 'round again, do a check activities.checkJobStillActive(); } } finally { os.close(); } deleteData(documentIdentifier); synchronized (this) { cacheData.put(documentIdentifier, new DocumentData(tempFile, contentType)); } return checkSum; } catch (IOException e) { ManifoldCF.deleteFile(tempFile); throw e; } catch (ServiceInterruption e) { ManifoldCF.deleteFile(tempFile); throw e; } catch (Error e) { ManifoldCF.deleteFile(tempFile); throw e; } } catch (java.net.SocketTimeoutException e) { throw new ManifoldCFException("Socket timeout exception creating temporary file: " + e.getMessage(), e); } catch (ConnectTimeoutException e) { throw new ManifoldCFException( "Socket connect timeout exception creating temporary file: " + e.getMessage(), e); } catch (InterruptedIOException e) { throw new ManifoldCFException("Interrupted: " + e.getMessage(), ManifoldCFException.INTERRUPTED); } catch (IOException e) { throw new ManifoldCFException("IO exception creating temporary file: " + e.getMessage(), e); } } /** Fetch binary data length. *@param documentIdentifier is the document identifier. *@return the length. */ public long getDataLength(String documentIdentifier) throws ManifoldCFException { DocumentData f; synchronized (this) { f = cacheData.get(documentIdentifier); } if (f == null) return 0L; return f.getData().length(); } /** Fetch binary data entry from the cache. *@param documentIdentifier is the document identifier (url). *@return a binary data stream. */ public InputStream getData(String documentIdentifier) throws ManifoldCFException { DocumentData f; synchronized (this) { f = cacheData.get(documentIdentifier); } if (f == null) return null; try { return new FileInputStream(f.getData()); } catch (IOException e) { throw new ManifoldCFException("IO exception getting data length: " + e.getMessage(), e); } } /** Get the content type. *@param documentIdentifier is the document identifier. *@return the content type, or null if there is none. */ public String getContentType(String documentIdentifier) { DocumentData dd; synchronized (this) { dd = cacheData.get(documentIdentifier); } if (dd == null) return null; return dd.getContentType(); } /** Delete specified item of data. *@param documentIdentifier is the document identifier (url). */ public void deleteData(String documentIdentifier) { DocumentData f; synchronized (this) { f = cacheData.remove(documentIdentifier); } if (f != null) { ManifoldCF.deleteFile(f.getData()); } } // Protected classes /** This class represents everything we need to know about a document that's getting passed from the * getDocumentVersions() phase to the processDocuments() phase. */ protected static class DocumentData { /** The cache file for the data */ protected File data; /** The content-type header value */ protected String contentType; /** Constructor. */ public DocumentData(File data, String contentType) { this.data = data; this.contentType = contentType; } /** Get the data */ public File getData() { return data; } /** Get the contentType */ public String getContentType() { return contentType; } } }