List of usage examples for java.io RandomAccessFile close
public void close() throws IOException
From source file:net.ontopia.infoset.content.FileContentStore.java
private void allocateNewBlock() throws ContentStoreException { RandomAccessFile out = null; boolean exception_thrown = false; try {// w w w . j a v a 2 s .co m out = new RandomAccessFile(key_file, "rws"); for (int i = 0; i < MAX_SPINS; i++) { // acquire exclusive lock FileLock l = out.getChannel().tryLock(); if (l == null) { // wait a little before trying again try { Thread.sleep(SPIN_TIMEOUT); } catch (InterruptedException e) { } continue; } else { try { // allocate new key int old_key; int new_key; String content = null; if (out.length() == 0) { old_key = 0; new_key = old_key + KEY_BLOCK_SIZE; } else { try { content = out.readUTF(); old_key = Integer.parseInt(content); new_key = old_key + KEY_BLOCK_SIZE; } catch (NumberFormatException e) { if (content.length() > 100) content = content.substring(0, 100) + "..."; throw new ContentStoreException( "Content store key file corrupted. Contained: '" + content + "'"); } } // truncate key file and write out new key out.seek(0); out.writeUTF(Integer.toString(new_key)); end_of_key_block = new_key; last_key = old_key; return; } finally { // release file lock try { l.release(); } catch (Throwable t) { throw new ContentStoreException("Could not release key file lock.", t); } } } } throw new ContentStoreException("Block allocation timed out."); } catch (ContentStoreException e) { exception_thrown = true; throw e; } catch (Throwable t) { exception_thrown = true; throw new ContentStoreException(t); } finally { if (out != null) { try { out.close(); } catch (IOException e) { if (!exception_thrown) throw new ContentStoreException("Problems occurred when closing content store.", e); } } } }
From source file:org.apache.james.mailrepository.file.MBoxMailRepository.java
/** * @see org.apache.james.mailrepository.api.MailRepository#remove(Collection) */// w ww . j ava2s. c om public void remove(final Collection<Mail> mails) { if ((getLogger().isDebugEnabled())) { String logBuffer = this.getClass().getName() + " Removing entry for key " + mails; getLogger().debug(logBuffer); } // The plan is as follows: // Attempt to locate the message in the file // by reading through the // once we've done that then seek to the file try { RandomAccessFile ins = new RandomAccessFile(mboxFile, "r"); // The // source final RandomAccessFile outputFile = new RandomAccessFile(mboxFile + WORKEXT, "rw"); // The // destination parseMboxFile(ins, new MessageAction() { public boolean isComplete() { return false; } public MimeMessage messageAction(String messageSeparator, String bodyText, long messageStart) { // Write out the messages as we go, until we reach the key // we want try { String currentKey = generateKeyValue(bodyText); boolean foundKey = false; Iterator<Mail> mailList = mails.iterator(); String key; while (mailList.hasNext()) { // Attempt to find the current key in the array key = mailList.next().getName(); if (key.equals(currentKey)) { // Don't write the message to disk foundKey = true; break; } } if (!foundKey) { // We didn't find the key in the array so we will // keep it outputFile.writeBytes(messageSeparator + "\n"); outputFile.writeBytes(bodyText); } } catch (NoSuchAlgorithmException e) { getLogger().error("MD5 not supported! ", e); } catch (IOException e) { getLogger().error("Unable to write file (General I/O problem) " + mboxFile, e); } return null; } }); ins.close(); outputFile.close(); // Delete the old mbox file File mbox = new File(mboxFile); FileUtils.forceDelete(mbox); // And rename the lock file to be the new mbox mbox = new File(mboxFile + WORKEXT); if (!mbox.renameTo(new File(mboxFile))) { throw new IOException("Failed to rename file " + mbox + " -> " + mboxFile); } // Now delete the keys in mails from the main hash Iterator<Mail> mailList = mails.iterator(); String key; while (mailList.hasNext()) { // Attempt to find the current key in the array key = mailList.next().getName(); mList.remove(key); } } catch (FileNotFoundException e) { getLogger().error("Unable to save(open) file (File not found) " + mboxFile, e); } catch (IOException e) { getLogger().error("Unable to write file (General I/O problem) " + mboxFile, e); } }
From source file:org.commoncrawl.service.listcrawler.CrawlList.java
void updateSubDomainQueueStatus(String rootDomainName, int deltaQueuedCount) throws IOException { long domainHash = URLFingerprint.generate64BitURLFPrint(rootDomainName); synchronized (_subDomainMetadataFile) { CrawlListMetadata metadata = new CrawlListMetadata(); RandomAccessFile file = new RandomAccessFile(_subDomainMetadataFile, "rw"); try {/*from w w w .jav a2s .c om*/ int dataOffset = getOffsetForSubDomainData(domainHash); if (dataOffset == 0) { throw new IOException("Data Offset Zero for host:" + rootDomainName); } file.seek(dataOffset); metadata.readFields(file); // set the data offset on the way out so that updates write to the proper location metadata.setQueuedItemCount(metadata.getQueuedItemCount() + deltaQueuedCount); // ok reseek to data offset file.seek(dataOffset); // rewrite the data structure metadata.write(file); } finally { file.close(); } } }
From source file:FileBaseDataMap.java
/** * put Method.<br>/* w w w . j av a 2 s.c o m*/ * * @param key * @param value * @param hashCode This is a key value hash code */ public void put(String key, String value, int hashCode) { try { File file = dataFileList[hashCode % numberOfDataFiles]; StringBuffer buf = new StringBuffer(this.fillCharacter(key, keyDataLength)); buf.append(this.fillCharacter(value, oneDataLength)); CacheContainer accessor = (CacheContainer) innerCache.get(file.getAbsolutePath()); RandomAccessFile raf = null; BufferedWriter wr = null; if (accessor == null || accessor.isClosed == true) { raf = new RandomAccessFile(file, "rwd"); wr = new BufferedWriter(new FileWriter(file, true)); accessor = new CacheContainer(); accessor.raf = raf; accessor.wr = wr; accessor.file = file; innerCache.put(file.getAbsolutePath(), accessor); } else { raf = accessor.raf; wr = accessor.wr; } // KeyData Write File for (int tryIdx = 0; tryIdx < 2; tryIdx++) { try { // Key?? long dataLineNo = this.getLinePoint(key, raf); if (dataLineNo == -1) { wr.write(buf.toString()); wr.flush(); // The size of an increment this.totalSize.getAndIncrement(); } else { // ?????1 boolean increMentFlg = false; if (this.get(key, hashCode) == null) increMentFlg = true; raf.seek(dataLineNo * (lineDataSize)); raf.write(buf.toString().getBytes(), 0, lineDataSize); if (increMentFlg) this.totalSize.getAndIncrement(); } break; } catch (IOException ie) { // IOException???1???? if (tryIdx == 1) throw ie; try { if (raf != null) raf.close(); if (wr != null) wr.close(); raf = new RandomAccessFile(file, "rwd"); wr = new BufferedWriter(new FileWriter(file, true)); accessor = new CacheContainer(); accessor.raf = raf; accessor.wr = wr; accessor.file = file; innerCache.put(file.getAbsolutePath(), accessor); } catch (Exception e) { throw e; } } } } catch (Exception e2) { e2.printStackTrace(); } }
From source file:org.commoncrawl.service.listcrawler.CrawlList.java
/** * Initialize a new CrawlList object from a given input stream of urls * //from ww w. j av a 2s . c o m * @param manager - reference to the crawl history log manager * @param urlInputStream - the input stream containing the list of urls that we should add to this list ... * @throws IOException */ public CrawlList(CrawlHistoryStorage manager, long listId, File sourceURLFile, int refreshInterval) throws IOException { _manager = manager; _listState = LoadState.REALLY_LOADING; // initialize a new list id _listId = listId; LOG.info("*** LIST:" + getListId() + " LOADING FROM SOURCE FILE:" + sourceURLFile.getAbsolutePath()); //establish file names initializeListFileNames(); sourceURLFile.renameTo(_listURLDataFile); FileInputStream urlInputStream = new FileInputStream(_listURLDataFile); try { // set we will use to hold all fingerprints generated TreeSet<URLFP> urlSet = new TreeSet<URLFP>(); // create temp files ... File spillOutputFile = File.createTempFile("spillOut", Long.toString(_listId)); // create mergesortspillwriter SequenceFileSpillWriter<URLFP, ProxyCrawlHistoryItem> spillwriter = new SequenceFileSpillWriter<URLFP, ProxyCrawlHistoryItem>( FileSystem.getLocal(CrawlEnvironment.getHadoopConfig()), CrawlEnvironment.getHadoopConfig(), new Path(spillOutputFile.getAbsolutePath()), URLFP.class, ProxyCrawlHistoryItem.class, null, false); try { MergeSortSpillWriter<URLFP, ProxyCrawlHistoryItem> merger = new MergeSortSpillWriter<URLFP, ProxyCrawlHistoryItem>( CrawlEnvironment.getHadoopConfig(), spillwriter, FileSystem.getLocal(CrawlEnvironment.getHadoopConfig()), new Path(manager.getLocalDataDir().getAbsolutePath()), null, new RawKeyValueComparator<URLFP, ProxyCrawlHistoryItem>() { DataInputBuffer _key1Buffer = new DataInputBuffer(); DataInputBuffer _key2Buffer = new DataInputBuffer(); @Override public int compareRaw(byte[] key1Data, int key1Offset, int key1Length, byte[] key2Data, int key2Offset, int key2Length, byte[] value1Data, int value1Offset, int value1Length, byte[] value2Data, int value2Offset, int value2Length) throws IOException { _key1Buffer.reset(key1Data, key1Offset, key1Length); _key2Buffer.reset(key2Data, key2Offset, key2Length); _key1Buffer.skip(2); // skip verison, and 1 byte id _key2Buffer.skip(2); // skip verison, and 1 byte id int domainHash1 = WritableUtils.readVInt(_key1Buffer); int domainHash2 = WritableUtils.readVInt(_key2Buffer); _key1Buffer.skip(1); // skip 1 byte id _key2Buffer.skip(1); // skip 1 byte id long fingerprint1 = WritableUtils.readVLong(_key1Buffer); long fingerprint2 = WritableUtils.readVLong(_key2Buffer); int result = ((Integer) domainHash1).compareTo(domainHash2); if (result == 0) { result = ((Long) fingerprint1).compareTo(fingerprint2); } return result; } @Override public int compare(URLFP key1, ProxyCrawlHistoryItem value1, URLFP key2, ProxyCrawlHistoryItem value2) { return key1.compareTo(key2); } }, URLFP.class, ProxyCrawlHistoryItem.class, false, null); try { LOG.info("*** LIST:" + getListId() + " Starting Scan of URLS In List"); BufferedReader reader = new BufferedReader( new InputStreamReader(urlInputStream, Charset.forName("UTF-8"))); String line = null; int lineNumber = 0; ProxyCrawlHistoryItem item = new ProxyCrawlHistoryItem(); while ((line = reader.readLine()) != null) { ++lineNumber; if (line.length() != 0 && !line.startsWith("#")) { URLFP fingerprint = URLUtils.getURLFPFromURL(line, true); if (fingerprint != null) { if (!urlSet.contains(fingerprint)) { // and add fingerprint to set urlSet.add(fingerprint); // initialize item item.clear(); item.setOriginalURL(line); // and spill to merger / sorter .. merger.spillRecord(fingerprint, item); } } else { LOG.error("*** LIST:" + getListId() + " Invalid URL Encounered at Line:" + lineNumber + " URL" + line); } } } LOG.info("*** LIST:" + getListId() + " Completed Scan of:" + urlSet.size() + " URLS"); } finally { merger.close(); } } finally { if (spillwriter != null) spillwriter.close(); } LOG.info("*** LIST:" + getListId() + " Generating BloomFilter for:" + urlSet.size() + " keys"); // generate bloom filter ... _bloomFilter = new URLFPBloomFilter(urlSet.size(), 7, 10); for (URLFP fingerprint : urlSet) { _bloomFilter.add(fingerprint); } LOG.info("*** LIST:" + getListId() + " Serializing BloomFilter"); // serialize it FileOutputStream bloomFilterStream = new FileOutputStream(_bloomFilterData); try { _bloomFilter.serialize(bloomFilterStream); } finally { bloomFilterStream.flush(); bloomFilterStream.close(); } LOG.info("*** LIST:" + getListId() + " Starting Read of Merged Sequence File:" + spillOutputFile); // now initialize value map and string maps based on output sequence file ... SequenceFile.Reader reader = new SequenceFile.Reader( FileSystem.getLocal(CrawlEnvironment.getHadoopConfig()), new Path(spillOutputFile.getAbsolutePath()), CrawlEnvironment.getHadoopConfig()); LOG.info("*** LIST:" + getListId() + " PRE-ALLOCATING FIXED DATA BUFFER OF SIZE:" + (urlSet.size() * OnDiskCrawlHistoryItem.ON_DISK_SIZE)); // OK, Allocate room for fixed data file upfront DataOutputBuffer valueStream = new DataOutputBuffer( urlSet.size() * OnDiskCrawlHistoryItem.ON_DISK_SIZE); LOG.info("*** LIST:" + getListId() + " ALLOCATION SUCCEEDED"); try { //DataOutputStream valueStream = new DataOutputStream(new FileOutputStream(_fixedDataFile)); RandomAccessFile stringsStream = new RandomAccessFile(_variableDataFile, "rw"); try { URLFP urlFP = new URLFP(); ProxyCrawlHistoryItem item = new ProxyCrawlHistoryItem(); // read fingerprints ... while (reader.next(urlFP, item)) { // write out fixed data structure and strings writeInitialOnDiskItem(urlFP, item, valueStream, stringsStream); } } finally { //valueStream.flush(); //valueStream.close(); stringsStream.close(); } } finally { reader.close(); } LOG.info("*** LIST:" + getListId() + " Finished Writing Initial Values to Disk"); LOG.info("*** LIST:" + getListId() + " FIXED DATA BUFFER OF SIZE:" + valueStream.getLength() + " EXCEPECTED SIZE:" + (urlSet.size() * OnDiskCrawlHistoryItem.ON_DISK_SIZE)); if (valueStream.getLength() != (urlSet.size() * OnDiskCrawlHistoryItem.ON_DISK_SIZE)) { throw new IOException("Final FixedItemData Buffer Size:" + valueStream.getLength() + " != URLSetSize:" + (urlSet.size() * OnDiskCrawlHistoryItem.ON_DISK_SIZE)); } // initialize temp data buffer variables _tempFixedDataBuffer = valueStream.getData(); _tempFixedDataBufferSize = valueStream.getLength(); // update metadata _metadata.setRefreshInterval(refreshInterval); _metadata.setUrlCount(urlSet.size()); // setup version _metadata.setVersion(1); // and write to disk writeMetadataToDisk(); // mark state as loaded ... _listState = LoadState.LOADED; LOG.info("*** LIST:" + getListId() + " SYNCING"); // reconcile with history log _manager.syncList(this.getListId(), urlSet, this); LOG.info("*** LIST:" + getListId() + " SYNC COMPLETE"); // write metdata to disk again writeMetadataToDisk(); LOG.info("*** LIST:" + getListId() + " FLUSHING FIXED DATA"); // and finally flush fixed data to disk FileOutputStream finalDataStream = new FileOutputStream(_fixedDataFile); try { synchronized (this) { int blockSize = 1 << 20; long bytesCopied = 0; for (int offset = 0; offset < _tempFixedDataBufferSize; offset += blockSize) { int bytesToCopy = Math.min(blockSize, _tempFixedDataBufferSize - offset); finalDataStream.write(_tempFixedDataBuffer, offset, bytesToCopy); bytesCopied += bytesToCopy; } // validate bytes copied if (bytesCopied != _tempFixedDataBufferSize) { throw new IOException("Buffer Size:" + _tempFixedDataBufferSize + " Does not Match BytesCopied:" + bytesCopied); } // ok release the buffer _tempFixedDataBuffer = null; _tempFixedDataBufferSize = 0; LOG.info("*** LIST:" + getListId() + " FIXED DATA FLUSH COMPLETE"); } } finally { finalDataStream.flush(); finalDataStream.close(); } // load sub domain metadata from disk ... loadSubDomainMetadataFromDisk(); } catch (IOException e) { LOG.error("*** LIST:" + getListId() + " Crawl List Initialization Failed With Exception:" + CCStringUtils.stringifyException(e)); _fixedDataFile.delete(); _variableDataFile.delete(); _bloomFilterData.delete(); _listState = LoadState.ERROR; throw e; } finally { urlInputStream.close(); } }
From source file:MSUmpire.SpectrumParser.mzXMLParser.java
private List<MzXMLthreadUnit> ParseScans(final BitSet IncludedScans) { List<MzXMLthreadUnit> ScanList = new ArrayList<>(); ArrayList<ForkJoinTask<?>> futures = new ArrayList<>(); final ForkJoinPool fjp = new ForkJoinPool(NoCPUs); Iterator<Entry<Integer, Long>> iter = ScanIndex.entrySet().iterator(); Entry<Integer, Long> ent = iter.next(); long currentIdx = ent.getValue(); int nextScanNo = ent.getKey(); final RandomAccessFile fileHandler; try {//from www. j a va2 s .c o m fileHandler = new RandomAccessFile(filename, "r"); } catch (FileNotFoundException e) { throw new RuntimeException(e); } byte[] buffer = new byte[1 << 10]; if (step == -1) step = fjp.getParallelism() * 32; while (iter.hasNext()) { ent = iter.next(); long startposition = currentIdx; long nexposition = ent.getValue(); int currentScanNo = nextScanNo; nextScanNo = ent.getKey(); currentIdx = nexposition; if (IncludedScans.get(currentScanNo)) { try { final int bufsize = (int) (nexposition - startposition); if (buffer.length < bufsize) buffer = new byte[Math.max(bufsize, buffer.length << 1)]; // byte[] buffer = new byte[bufsize]; // RandomAccessFile fileHandler = new RandomAccessFile(filename, "r"); fileHandler.seek(startposition); fileHandler.read(buffer, 0, bufsize); // fileHandler.close(); // String xmltext = new String(buffer); String xmltext = new String(buffer, 0, bufsize, StandardCharsets.ISO_8859_1); if (ent.getKey() == Integer.MAX_VALUE) { xmltext = xmltext.replaceAll("</msRun>", ""); } boolean ReadPeak = true; final MzXMLthreadUnit unit = new MzXMLthreadUnit(xmltext, parameter, datatype, ReadPeak); futures.add(fjp.submit(unit)); ScanList.add(unit); if ((ScanList.size() % step) == 0) { futures.get(futures.size() - step).get(); if (iter.hasNext() && fjp.getActiveThreadCount() < fjp.getParallelism()) { step *= 2; // System.out.println("MzXMLthreadUnit: fjp.getActiveThreadCount()\t" + fjp.getActiveThreadCount()+"\t"+step); } } } catch (Exception ex) { Logger.getRootLogger().error(ExceptionUtils.getStackTrace(ex)); } } } try { fileHandler.close(); } catch (IOException ex) { throw new RuntimeException(ex); } fjp.shutdown(); try { fjp.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); } catch (InterruptedException ex) { throw new RuntimeException(ex); } // for (MzXMLthreadUnit unit : ScanList) { // executorPool.execute(unit); // } // executorPool.shutdown(); // // try { // executorPool.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); // } catch (InterruptedException e) { // Logger.getRootLogger().info("interrupted.."); // } return ScanList; }
From source file:org.commoncrawl.service.listcrawler.CrawlList.java
/** * update list state of a recently crawled item * // www . j ava 2 s . c o m * @param fingerprint - the fingerprint of the updated item * @param newData - the updated crawl history data for the given item * @throws IOException */ @Override public void updateItemState(URLFP fingerprint, ProxyCrawlHistoryItem newData) throws IOException { if (_listState == LoadState.LOADED) { // check for membership ... if (_bloomFilter.isPresent(fingerprint)) { //LOG.info("UpdateItemState Called for URL:" + newData.getOriginalURL() + " List:" + getListId()); //LOG.info("UpdateItemState Loading OnDisk Item for URL:" + newData.getOriginalURL() + " List:" + getListId()); // extract existing item from disk OnDiskCrawlHistoryItem originalItem = loadOnDiskItemForURLFP(fingerprint); //if present (null if false cache hit) if (originalItem != null) { // build an on disk item data structure for any potential changes ... OnDiskCrawlHistoryItem newItem = onDiskItemFromHistoryItem(fingerprint, newData); // set inital offset information newItem._fileOffset = originalItem._fileOffset; newItem._stringsOffset = originalItem._stringsOffset; // LOG.info("UpdateItemState Comparing OnDisk Item to New Item for URL:" + newData.getOriginalURL() + " List:" + getListId()); // compare the two items ... if (!newItem.equals(originalItem)) { //LOG.info("UpdateItemState Items Don't Match for URL:" + newData.getOriginalURL() + " List:" + getListId()); // ok items do not match ... figure out if strings are different ... if (newItem._stringsCRC != originalItem._stringsCRC) { RandomAccessFile stringsFile = new RandomAccessFile(_variableDataFile, "rw"); try { // seek to end stringsFile.seek(stringsFile.length()); // update offset info newItem._stringsOffset = stringsFile.length(); // write out string data length WritableUtils.writeVInt(stringsFile, _stringBuffer1.getLength()); // write strings to log file stringsFile.write(_stringBuffer1.getData(), 0, _stringBuffer1.getLength()); } finally { stringsFile.close(); } } // otherwise take the offset from old item else { newItem._stringsOffset = originalItem._stringsOffset; } //LOG.info("Opening Data File for OnDiskItem load for Fingerprint:" + newItem._urlFingerprint); // ok, different paths depending on wether this is an in memory update or not ... boolean wroteToMemory = false; synchronized (this) { if (_tempFixedDataBuffer != null) { wroteToMemory = true; // reset output buffer _tempOutputBuffer.reset(); // serizlie to output buffer newItem.serialize(_tempOutputBuffer); // copy to appropriate location System.arraycopy(_tempOutputBuffer.getData(), 0, _tempFixedDataBuffer, (int) originalItem._fileOffset, OnDiskCrawlHistoryItem.ON_DISK_SIZE); } } if (!wroteToMemory) { // write to disk RandomAccessFile file = new RandomAccessFile(_fixedDataFile, "rw"); try { while (true) { try { //LOG.info("*** TRYING UPDATE LOCK FOR OFFSET:" + originalItem._fileOffset); FileLock lock = file.getChannel().tryLock(originalItem._fileOffset, OnDiskCrawlHistoryItem.ON_DISK_SIZE, false); try { //LOG.info("*** GOT UPDATE LOCK FOR OFFSET:" + originalItem._fileOffset); file.seek(originalItem._fileOffset); newItem.serialize(file); //LOG.info("Updated Data File for OnDiskItem for Fingerprint:" + originalItem._urlFingerprint); break; } finally { //LOG.info("*** RELEASED UPDATE LOCK FOR OFFSET:" + originalItem._fileOffset); lock.release(); } } catch (OverlappingFileLockException e) { LOG.error("###LockConflict(RETRY):" + CCStringUtils.stringifyException(e)); } } } finally { file.close(); } } // ok now update metadata ... synchronized (_metadata) { int updateFlags = calculateUpdateFlags(originalItem, newItem); if (updateFlags != 0) { int metadataDirtyFlags = updateMetadata(newItem, _metadata, 0); // only write metadata to disk if temp data buffer is null if (metadataDirtyFlags != 0 && !wroteToMemory) { if ((metadataDirtyFlags & MetadataUpdateFlag_ModifiedCrawlStatus) != 0) { _metadata.setQueuedItemCount(_metadata.getQueuedItemCount() - 1); } writeMetadataToDisk(); } // if not writing to memory then update subdomain metadata if (!wroteToMemory) { synchronized (_subDomainMetadataFile) { CrawlListMetadata subDomainMetadata = getSubDomainMetadataByURL( newData.getOriginalURL()); int subDomainMetadataDirtyFlags = updateMetadata(newItem, subDomainMetadata, processFileOffsets); if (subDomainMetadataDirtyFlags != 0 && !wroteToMemory) { if ((subDomainMetadataDirtyFlags & MetadataUpdateFlag_ModifiedCrawlStatus) != 0) { subDomainMetadata.setQueuedItemCount( subDomainMetadata.getQueuedItemCount() - 1); } writeSubDomainMetadataToDisk(subDomainMetadata); } } } } } synchronized (this) { if (_eventListener != null) { _eventListener.itemUpdated(fingerprint); } } } } } } }
From source file:com.turn.griffin.data.GriffinUploadTask.java
private void uploadFile(FileInfo fileInfo, BitSet availableBlockBitmap) { String filename = fileInfo.getFilename(); long fileVersion = fileInfo.getVersion(); long blockCount = fileInfo.getBlockCount(); long blockSize = fileInfo.getBlockSize(); byte[] buffer = new byte[(int) blockSize]; GriffinLibCacheUtil libCacheManager = dataManager.getLibCacheManager().get(); String dataTopicNameForProducer = GriffinKafkaTopicNameUtil.getDataTopicNameForProducer(filename, fileVersion);// w w w. j av a 2s . com GriffinProducer producer = null; try { String libCacheUploadFilePath = libCacheManager.getUploadFilePath(fileInfo); RandomAccessFile libCacheUploadFile = new RandomAccessFile(libCacheUploadFilePath, "r"); producer = new GriffinProducer(GriffinModule.BROKERS); logger.info(String.format("Starting to push %s", fileInfo.toString().replaceAll(System.getProperty("line.separator"), " "))); int uploadAttempts = 0; while (availableBlockBitmap.nextClearBit(0) != blockCount) { /* If a new version has arrived abort uploading older version */ if (!libCacheManager.isLatestGlobalVersion(fileInfo)) { logger.info( String.format("Aborting upload for %s version %s as a newer version is now available.", filename, fileVersion)); break; } if (uploadAttempts >= maxUploadAttempts) { logger.warn(String.format("Unable to upload %s version %s after %s attempts", filename, fileVersion, uploadAttempts)); String subject = String.format("WARNING: GriffinUploadTask failed for blob:%s", filename); String body = String.format( "Action: GriffinUploadTask failed for blob:%s version:%s%n" + "Reason: Unable to upload after %s attempts%n", filename, fileVersion, uploadAttempts); GriffinModule.emailAlert(subject, body); break; } int blockToUpload = availableBlockBitmap.nextClearBit(0); libCacheUploadFile.seek(blockToUpload * blockSize); int bytesRead = libCacheUploadFile.read(buffer); DataMessage msg = DataMessage.newBuilder().setBlockSeqNo(blockToUpload).setByteCount(bytesRead) .setData(ByteString.copyFrom(buffer)).build(); try { producer.send(dataTopicNameForProducer, DigestUtils.md5Hex(buffer), msg); availableBlockBitmap.set(blockToUpload); uploadAttempts = 0; } catch (FailedToSendMessageException ftsme) { /* Retry the same block again */ logger.warn(String.format("Unable to send block %s for file: %s version: %s " + "due to FailedToSendMessageException", blockToUpload, filename, fileVersion)); uploadAttempts++; } catch (Exception e) { logger.warn(String.format("Unable to send block %s for file: %s version: %s", blockToUpload, filename, fileVersion), e); logger.warn("Exception", e); uploadAttempts++; } } logger.info(String.format("Ending file upload for file %s version %s to %s", filename, fileVersion, dataTopicNameForProducer)); libCacheUploadFile.close(); } catch (IOException | RuntimeException e) { logger.error(String.format("Unable to upload file %s to %s", filename, dataTopicNameForProducer), e); String subject = String.format("WARNING: GriffinUploadTask failed for blob:%s", filename); String body = String.format( "Action: GriffinUploadTask failed for blob:%s version:%s%n" + "Reason: Exception in GriffinUploadTask%n %s", filename, fileVersion, Throwables.getStackTraceAsString(e)); GriffinModule.emailAlert(subject, body); } finally { if (producer != null) { producer.shutdown(); } } }
From source file:com.cyberway.issue.crawler.fetcher.FetchHTTP.java
/** * Load cookies from a file before the first fetch. * <p>//from www . j av a 2 s .c om * The file is a text file in the Netscape's 'cookies.txt' file format.<br> * Example entry of cookies.txt file:<br> * <br> * www.archive.org FALSE / FALSE 1074567117 details-visit texts-cralond<br> * <br> * Each line has 7 tab-separated fields:<br> * <li>1. DOMAIN: The domain that created and have access to the cookie * value. * <li>2. FLAG: A TRUE or FALSE value indicating if hosts within the given * domain can access the cookie value. * <li>3. PATH: The path within the domain that the cookie value is valid * for. * <li>4. SECURE: A TRUE or FALSE value indicating if to use a secure * connection to access the cookie value. * <li>5. EXPIRATION: The expiration time of the cookie value (unix style.) * <li>6. NAME: The name of the cookie value * <li>7. VALUE: The cookie value * * @param cookiesFile file in the Netscape's 'cookies.txt' format. */ public void loadCookies(String cookiesFile) { // Do nothing if cookiesFile is not specified. if (cookiesFile == null || cookiesFile.length() <= 0) { return; } RandomAccessFile raf = null; try { raf = new RandomAccessFile(cookiesFile, "r"); String[] cookieParts; String line; Cookie cookie = null; while ((line = raf.readLine()) != null) { // Line that starts with # is commented line, therefore skip it. if (!line.startsWith("#")) { cookieParts = line.split("\\t"); if (cookieParts.length == 7) { // Create cookie with not expiration date (-1 value). // TODO: add this as an option. cookie = new Cookie(cookieParts[0], cookieParts[5], cookieParts[6], cookieParts[2], -1, Boolean.valueOf(cookieParts[3]).booleanValue()); if (cookieParts[1].toLowerCase().equals("true")) { cookie.setDomainAttributeSpecified(true); } else { cookie.setDomainAttributeSpecified(false); } this.http.getState().addCookie(cookie); logger.fine("Adding cookie: " + cookie.toExternalForm()); } } } } catch (FileNotFoundException e) { // We should probably throw FatalConfigurationException. System.out.println("Could not find file: " + cookiesFile + " (Element: " + ATTR_LOAD_COOKIES + ")"); } catch (IOException e) { // We should probably throw FatalConfigurationException. e.printStackTrace(); } finally { try { if (raf != null) { raf.close(); } } catch (IOException e) { e.printStackTrace(); } } }
From source file:au.org.ala.layers.intersect.Grid.java
/** * for grid cutter// w ww. ja va 2 s. c o m * <p/> * writes out a list of double (same as getGrid() returns) to a file * <p/> * byteorderlsb * data type, FLOAT * * @param newfilename * @param dfiltered */ public void writeGrid(String newfilename, double[] dfiltered, double xmin, double ymin, double xmax, double ymax, double xres, double yres, int nrows, int ncols) { int size, i, length = dfiltered.length; double maxvalue = Double.MAX_VALUE * -1; double minvalue = Double.MAX_VALUE; //write data as whole file RandomAccessFile afile = null; try { //read of random access file can throw an exception afile = new RandomAccessFile(newfilename + ".gri", "rw"); size = 4; byte[] b = new byte[size * length]; ByteBuffer bb = ByteBuffer.wrap(b); if (byteorderLSB) { bb.order(ByteOrder.LITTLE_ENDIAN); } else { bb.order(ByteOrder.BIG_ENDIAN); } for (i = 0; i < length; i++) { if (Double.isNaN(dfiltered[i])) { bb.putFloat((float) noDataValueDefault); } else { if (minvalue > dfiltered[i]) { minvalue = dfiltered[i]; } if (maxvalue < dfiltered[i]) { maxvalue = dfiltered[i]; } bb.putFloat((float) dfiltered[i]); } } afile.write(b); } catch (Exception e) { logger.error("error writing grid file", e); } finally { if (afile != null) { try { afile.close(); } catch (Exception e) { logger.error(e.getMessage(), e); } } } writeHeader(newfilename, xmin, ymin, xmin + xres * ncols, ymin + yres * nrows, xres, yres, nrows, ncols, minvalue, maxvalue, "FLT4BYTES", String.valueOf(noDataValueDefault)); }