List of usage examples for java.io RandomAccessFile length
public native long length() throws IOException;
From source file:org.commoncrawl.service.listcrawler.CrawlList.java
private OnDiskCrawlHistoryItem loadOnDiskItemForURLFP(URLFP fingerprint) throws IOException { // see if state is cached in memory ... boolean loadedFromMemory = false; synchronized (this) { if (_tempFixedDataBuffer != null) { loadedFromMemory = true;//w ww .j a v a 2 s.c o m int low = 0; int high = (int) (_tempFixedDataBufferSize / OnDiskCrawlHistoryItem.ON_DISK_SIZE) - 1; OnDiskCrawlHistoryItem itemOut = new OnDiskCrawlHistoryItem(); DataInputBuffer inputBuffer = new DataInputBuffer(); int iterationNumber = 0; while (low <= high) { ++iterationNumber; int mid = low + ((high - low) / 2); inputBuffer.reset(_tempFixedDataBuffer, 0, _tempFixedDataBufferSize); inputBuffer.skip(mid * OnDiskCrawlHistoryItem.ON_DISK_SIZE); // deserialize itemOut.deserialize(inputBuffer); // now compare it against desired hash value ... int comparisonResult = itemOut.compareFingerprints(fingerprint); if (comparisonResult > 0) high = mid - 1; else if (comparisonResult < 0) low = mid + 1; else { // cache offset itemOut._fileOffset = mid * OnDiskCrawlHistoryItem.ON_DISK_SIZE; // LOG.info("Found Match. Took:"+ iterationNumber + " iterations"); // and return item return itemOut; } } //LOG.error("Did Not Find Match For Domain:" + fingerprint.getDomainHash() + " URLFP:" + fingerprint.getUrlHash() + " Took:" + iterationNumber + " iterations"); } } if (!loadedFromMemory) { //load from disk //LOG.info("Opening Data File for OnDiskItem load for Fingerprint:" + fingerprint.getUrlHash()); RandomAccessFile file = new RandomAccessFile(_fixedDataFile, "rw"); // allocate buffer upfront byte[] onDiskItemBuffer = new byte[OnDiskCrawlHistoryItem.ON_DISK_SIZE]; DataInputBuffer inputStream = new DataInputBuffer(); //LOG.info("Opened Data File. Searching for match"); try { int low = 0; int high = (int) (file.length() / OnDiskCrawlHistoryItem.ON_DISK_SIZE) - 1; OnDiskCrawlHistoryItem itemOut = new OnDiskCrawlHistoryItem(); int iterationNumber = 0; while (low <= high) { ++iterationNumber; int mid = low + ((high - low) / 2); // seek to proper location file.seek(mid * OnDiskCrawlHistoryItem.ON_DISK_SIZE); // read the data structure file.readFully(onDiskItemBuffer, 0, onDiskItemBuffer.length); // map location in file //MappedByteBuffer memoryBuffer = file.getChannel().map(MapMode.READ_ONLY,mid * OnDiskCrawlHistoryItem.ON_DISK_SIZE,OnDiskCrawlHistoryItem.ON_DISK_SIZE); //DataInputStream inputStream = new DataInputStream(new ByteBufferInputStream(memoryBuffer)); inputStream.reset(onDiskItemBuffer, 0, OnDiskCrawlHistoryItem.ON_DISK_SIZE); // deserialize itemOut.deserialize(inputStream); // memoryBuffer = null; //inputStream = null; // now compare it against desired hash value ... int comparisonResult = itemOut.compareFingerprints(fingerprint); if (comparisonResult > 0) high = mid - 1; else if (comparisonResult < 0) low = mid + 1; else { // cache offset itemOut._fileOffset = mid * OnDiskCrawlHistoryItem.ON_DISK_SIZE; // LOG.info("Found Match. Took:"+ iterationNumber + " iterations"); // and return item return itemOut; } } //LOG.error("******Did Not Find Match For Domain:" + fingerprint.getDomainHash() + " URLFP:" + fingerprint.getUrlHash() + " Took:" + iterationNumber + " iterations"); //DEBUG ONLY ! // dumpFixedDataFile(); } finally { file.close(); } } return null; }
From source file:au.org.ala.layers.intersect.Grid.java
/** * @param points input array for longitude and latitude * double[number_of_points][2] and sorted latitude then longitude * @return array of .gri file values corresponding to the * points provided//from w w w. j a v a 2 s .c om */ public float[] getValues3(double[][] points, int bufferSize) { //confirm inputs since they come from somewhere else if (points == null || points.length == 0) { return null; } if (subgrids != null) { return getValuesSubgrids(points, bufferSize); } //use preloaded grid data if available Grid g = Grid.getLoadedGrid(filename); if (g != null && g.grid_data != null) { return g.getValues2(points); } int length = points.length; int size, i; byte[] b; RandomAccessFile afile = null; File f2 = new File(filename + ".GRI"); try { //read of random access file can throw an exception if (!f2.exists()) { afile = new RandomAccessFile(filename + ".gri", "r"); } else { afile = new RandomAccessFile(filename + ".GRI", "r"); } //do not cache subgrids (using getValues2) if (!subgrid && afile.length() < 80 * 1024 * 1024) { try { afile.close(); afile = null; } catch (Exception e) { } return getValues2(points); } byte[] buffer = new byte[bufferSize]; //must be multiple of 64 Long bufferOffset = afile.length(); float[] ret = new float[points.length]; //get cell numbers long[][] cells = new long[points.length][2]; for (int j = 0; j < points.length; j++) { if (Double.isNaN(points[j][0]) || Double.isNaN(points[j][1])) { cells[j][0] = -1; cells[j][1] = j; } else { cells[j][0] = getcellnumber(points[j][0], points[j][1]); cells[j][1] = j; } } java.util.Arrays.sort(cells, new Comparator<long[]>() { @Override public int compare(long[] o1, long[] o2) { if (o1[0] == o2[0]) { return o1[1] > o2[1] ? 1 : -1; } else { return o1[0] > o2[0] ? 1 : -1; } } }); if (datatype.equalsIgnoreCase("BYTE")) { size = 1; for (i = 0; i < length; i++) { if (i > 0 && cells[i - 1][0] == cells[i][0]) { ret[(int) cells[i][1]] = ret[(int) cells[i - 1][1]]; continue; } if (cells[i][0] >= 0) { ret[(int) cells[i][1]] = getByte(afile, buffer, bufferOffset, cells[i][0] * size); } else { ret[(int) cells[i][1]] = Float.NaN; } } } else if (datatype.equalsIgnoreCase("UBYTE")) { size = 1; for (i = 0; i < length; i++) { if (i > 0 && cells[i - 1][0] == cells[i][0]) { ret[(int) cells[i][1]] = ret[(int) cells[i - 1][1]]; continue; } if (cells[i][0] >= 0) { ret[(int) cells[i][1]] = getByte(afile, buffer, bufferOffset, cells[i][0] * size); if (ret[(int) cells[i][1]] < 0) { ret[(int) cells[i][1]] += 256; } } else { ret[(int) cells[i][1]] = Float.NaN; } } } else if (datatype.equalsIgnoreCase("SHORT")) { size = 2; b = new byte[size]; for (i = 0; i < length; i++) { if (i > 0 && cells[i - 1][0] == cells[i][0]) { ret[(int) cells[i][1]] = ret[(int) cells[i - 1][1]]; continue; } if (cells[i][0] >= 0) { bufferOffset = getBytes(afile, buffer, bufferOffset, cells[i][0] * (long) size, b); if (byteorderLSB) { ret[(int) cells[i][1]] = (short) (((0xFF & b[1]) << 8) | (b[0] & 0xFF)); } else { ret[(int) cells[i][1]] = (short) (((0xFF & b[0]) << 8) | (b[1] & 0xFF)); } } else { ret[(int) cells[i][1]] = Float.NaN; } } } else if (datatype.equalsIgnoreCase("INT")) { size = 4; b = new byte[size]; for (i = 0; i < length; i++) { if (i > 0 && cells[i - 1][0] == cells[i][0]) { ret[(int) cells[i][1]] = ret[(int) cells[i - 1][1]]; continue; } if (cells[i][0] >= 0) { bufferOffset = getBytes(afile, buffer, bufferOffset, cells[i][0] * (long) size, b); if (byteorderLSB) { ret[(int) cells[i][1]] = ((0xFF & b[3]) << 24) | ((0xFF & b[2]) << 16) + ((0xFF & b[1]) << 8) + (b[0] & 0xFF); } else { ret[(int) cells[i][1]] = ((0xFF & b[0]) << 24) | ((0xFF & b[1]) << 16) + ((0xFF & b[2]) << 8) + ((0xFF & b[3]) & 0xFF); } } else { ret[(int) cells[i][1]] = Float.NaN; } } } else if (datatype.equalsIgnoreCase("LONG")) { size = 8; b = new byte[size]; for (i = 0; i < length; i++) { if (i > 0 && cells[i - 1][0] == cells[i][0]) { ret[(int) cells[i][1]] = ret[(int) cells[i - 1][1]]; continue; } if (cells[i][0] >= 0) { bufferOffset = getBytes(afile, buffer, bufferOffset, cells[i][0] * (long) size, b); if (byteorderLSB) { ret[(int) cells[i][1]] = ((long) (0xFF & b[7]) << 56) + ((long) (0xFF & b[6]) << 48) + ((long) (0xFF & b[5]) << 40) + ((long) (0xFF & b[4]) << 32) + ((long) (0xFF & b[3]) << 24) + ((long) (0xFF & b[2]) << 16) + ((long) (0xFF & b[1]) << 8) + (0xFF & b[0]); } else { ret[(int) cells[i][1]] = ((long) (0xFF & b[0]) << 56) + ((long) (0xFF & b[1]) << 48) + ((long) (0xFF & b[2]) << 40) + ((long) (0xFF & b[3]) << 32) + ((long) (0xFF & b[4]) << 24) + ((long) (0xFF & b[5]) << 16) + ((long) (0xFF & b[6]) << 8) + (0xFF & b[7]); } } else { ret[(int) cells[i][1]] = Float.NaN; } } } else if (datatype.equalsIgnoreCase("FLOAT")) { size = 4; b = new byte[size]; for (i = 0; i < length; i++) { if (i > 0 && cells[i - 1][0] == cells[i][0]) { ret[(int) cells[i][1]] = ret[(int) cells[i - 1][1]]; continue; } if (cells[i][0] >= 0) { bufferOffset = getBytes(afile, buffer, bufferOffset, cells[i][0] * (long) size, b); ByteBuffer bb = ByteBuffer.wrap(b); if (byteorderLSB) { bb.order(ByteOrder.LITTLE_ENDIAN); } ret[(int) cells[i][1]] = bb.getFloat(); } else { ret[(int) cells[i][1]] = Float.NaN; } } } else if (datatype.equalsIgnoreCase("DOUBLE")) { size = 8; b = new byte[8]; for (i = 0; i < length; i++) { if (i > 0 && cells[i - 1][0] == cells[i][0]) { ret[(int) cells[i][1]] = ret[(int) cells[i - 1][1]]; continue; } if (cells[i][0] >= 0) { getBytes(afile, buffer, bufferOffset, cells[i][0] * (long) size, b); ByteBuffer bb = ByteBuffer.wrap(b); if (byteorderLSB) { bb.order(ByteOrder.LITTLE_ENDIAN); } ret[(int) cells[i][1]] = (float) bb.getDouble(); } else { ret[(int) cells[i][1]] = Float.NaN; } } } else { logger.error("datatype not supported in Grid.getValues: " + datatype); // / should not happen; catch anyway... for (i = 0; i < length; i++) { ret[i] = Float.NaN; } } //replace not a number for (i = 0; i < length; i++) { if ((float) ret[i] == (float) nodatavalue) { ret[i] = Float.NaN; } else { ret[i] *= rescale; } } return ret; } catch (Exception e) { logger.error("error getting grid file values", e); } finally { if (afile != null) { try { afile.close(); } catch (Exception e) { logger.error(e.getMessage(), e); } } } return null; }
From source file:org.commoncrawl.service.listcrawler.CrawlList.java
void loadSubDomainMetadataFromDisk() throws IOException { LOG.info("*** LIST:" + getListId() + " LOAD SUBDOMAIN METADATA FROM DISK ... "); if (_subDomainMetadataFile.exists()) { LOG.info("*** LIST:" + getListId() + " FILE EXISTS LOADING SUBDOMAIN DATA FROM DISK."); RandomAccessFile file = new RandomAccessFile(_subDomainMetadataFile, "rw"); DataInputBuffer inputBuffer = new DataInputBuffer(); byte fixedDataBlock[] = new byte[CrawlListMetadata.Constants.FixedDataSize]; try {/* w w w . ja v a2 s . c o m*/ // skip version file.read(); // read item count int itemCount = file.readInt(); LOG.info("*** LIST:" + getListId() + " SUBDOMAIN ITEM COUNT:" + itemCount); CrawlListMetadata newMetadata = new CrawlListMetadata(); TreeMap<Long, Integer> idToOffsetMap = new TreeMap<Long, Integer>(); for (int i = 0; i < itemCount; ++i) { long orignalPos = file.getFilePointer(); file.readFully(fixedDataBlock, 0, fixedDataBlock.length); inputBuffer.reset(fixedDataBlock, fixedDataBlock.length); try { newMetadata.deserialize(inputBuffer, new BinaryProtocol()); } catch (Exception e) { LOG.error("-----Failed to Deserialize Metadata at Index:" + i + " Exception:" + CCStringUtils.stringifyException(e)); } idToOffsetMap.put(newMetadata.getDomainHash(), (int) orignalPos); } // write lookup table _offsetLookupTable = new DataOutputBuffer(idToOffsetMap.size() * OFFSET_TABLE_ENTRY_SIZE); for (Map.Entry<Long, Integer> entry : idToOffsetMap.entrySet()) { _offsetLookupTable.writeLong(entry.getKey()); _offsetLookupTable.writeInt(entry.getValue()); } } finally { file.close(); } LOG.info("*** LIST:" + getListId() + " DONE LOADING SUBDOMAIN DATA FROM DISK"); } else { LOG.info("*** LIST:" + getListId() + " SUBDOMAIN METADATA DOES NOT EXIST! LOADING FROM SCRATCH"); RandomAccessFile fixedDataReader = new RandomAccessFile(_fixedDataFile, "rw"); RandomAccessFile stringDataReader = new RandomAccessFile(_variableDataFile, "rw"); try { //ok rebuild top level metadata as well _metadata.clear(); OnDiskCrawlHistoryItem item = new OnDiskCrawlHistoryItem(); int processedCount = 0; while (fixedDataReader.getFilePointer() != fixedDataReader.length()) { long position = fixedDataReader.getFilePointer(); // store offset in item item._fileOffset = position; // load from disk item.deserialize(fixedDataReader); try { // seek to string data stringDataReader.seek(item._stringsOffset); // and skip buffer length WritableUtils.readVInt(stringDataReader); // and read primary string String url = stringDataReader.readUTF(); // get metadata object for subdomain CrawlListMetadata subDomainMetadata = getTransientSubDomainMetadata(url); // increment url count subDomainMetadata.setUrlCount(subDomainMetadata.getUrlCount() + 1); // increment top level metadata count _metadata.setUrlCount(_metadata.getUrlCount() + 1); // update top level metadata .. updateMetadata(item, _metadata, 0); // update sub-domain metadata object from item data updateMetadata(item, subDomainMetadata, 0); ++processedCount; } catch (IOException e) { LOG.error("Exception Reading String Data For Item:" + (processedCount + 1)); LOG.error("Exception:" + CCStringUtils.stringifyException(e)); LOG.error("File Position:" + fixedDataReader.getFilePointer() + " StringsPointer:" + stringDataReader.getFilePointer()); } if (processedCount % 10000 == 0) { LOG.info("*** LIST:" + getListId() + " Processed:" + processedCount + " Items"); } } // ok commit top level metadata to disk as well writeMetadataToDisk(); } catch (IOException e) { LOG.error("Encountered Exception Queueing Items for List:" + _listId + " Exception:" + CCStringUtils.stringifyException(e)); LOG.error("File Position:" + fixedDataReader.getFilePointer() + " StringsPointer:" + stringDataReader.getFilePointer()); _queueState = QueueState.QUEUED; } finally { fixedDataReader.close(); stringDataReader.close(); } LOG.info("*** LIST:" + getListId() + " SUBDOMAIN METADATA REBUILT FROM LIST DATA . WRITING TO DISK"); // write metadat to disk writeInitialSubDomainMetadataToDisk(); LOG.info("*** LIST:" + getListId() + " SUBDOMAIN METADATA REBUILT FROM LIST DATA . WRITE COMPLETE"); } }
From source file:tvbrowser.ui.mainframe.MainFrame.java
@Override public void drop(DropTargetDropEvent dtde) { dtde.acceptDrop(dtde.getDropAction()); File[] files = getDragDropPlugins(dtde.getCurrentDataFlavors(), dtde.getTransferable()); try {//ww w. j a va2 s . c o m File tmpFile = File.createTempFile("plugins", ".txt"); StringBuilder alreadyInstalled = new StringBuilder(); StringBuilder notCompatiblePlugins = new StringBuilder(); for (File jarFile : files) { ClassLoader classLoader = null; try { URL[] urls = new URL[] { jarFile.toURI().toURL() }; classLoader = URLClassLoader.newInstance(urls, ClassLoader.getSystemClassLoader()); } catch (MalformedURLException exc) { } if (classLoader != null) { // Get the plugin name String pluginName = jarFile.getName(); pluginName = pluginName.substring(0, pluginName.length() - 4); try { String pluginId = "java." + pluginName.toLowerCase() + "." + pluginName; PluginProxy installedPlugin = PluginProxyManager.getInstance().getPluginForId(pluginId); TvDataServiceProxy service = TvDataServiceProxyManager.getInstance() .findDataServiceById(pluginName.toLowerCase() + '.' + pluginName); Class<?> pluginClass = classLoader.loadClass(pluginName.toLowerCase() + '.' + pluginName); Method getVersion = pluginClass.getMethod("getVersion", new Class[0]); Version version1 = null; try { version1 = (Version) getVersion.invoke(pluginClass, new Object[0]); } catch (Throwable t1) { t1.printStackTrace(); } if (installedPlugin != null && (installedPlugin.getInfo().getVersion().compareTo(version1) > 0 || (installedPlugin.getInfo().getVersion().compareTo(version1) == 0 && version1.isStable()))) { alreadyInstalled.append(installedPlugin.getInfo().getName()).append('\n'); } else if (service != null && (service.getInfo().getVersion().compareTo(version1) > 0 || (service.getInfo().getVersion().compareTo(version1) == 0 && version1.isStable()))) { alreadyInstalled.append(service.getInfo().getName()).append('\n'); } else { RandomAccessFile write = new RandomAccessFile(tmpFile, "rw"); String versionString = Integer.toString(version1.getMajor()) + '.' + (version1.getMinor() / 10) + (version1.getMinor() % 10) + '.' + version1.getSubMinor(); write.seek(write.length()); write.writeBytes("[plugin:" + pluginName + "]\n"); write.writeBytes("name_en=" + pluginName + "\n"); write.writeBytes("filename=" + jarFile.getName() + "\n"); write.writeBytes("version=" + versionString + "\n"); write.writeBytes("stable=" + version1.isStable() + "\n"); write.writeBytes("download=" + jarFile.toURI().toURL() + "\n"); write.writeBytes("category=unknown\n"); write.close(); } } catch (Exception e) { notCompatiblePlugins.append(jarFile.getName()).append("\n"); } } } if (alreadyInstalled.length() > 0) { showInfoTextMessage( mLocalizer.msg("update.alreadyInstalled", "The following Plugin in current version are already installed:"), alreadyInstalled.toString(), 400); } if (notCompatiblePlugins.length() > 0) { showInfoTextMessage( mLocalizer.msg("update.noTVBPlugin", "This following files are not TV-Browser Plugins:"), notCompatiblePlugins.toString(), 400); } if (tmpFile.length() > 0) { java.net.URL url = tmpFile.toURI().toURL(); SoftwareUpdater softwareUpdater = new SoftwareUpdater(url, false, true); mSoftwareUpdateItems = softwareUpdater.getAvailableSoftwareUpdateItems(); dtde.dropComplete(true); SoftwareUpdateDlg updateDlg = new SoftwareUpdateDlg(this, false, mSoftwareUpdateItems); updateDlg.setVisible(true); } else { dtde.rejectDrop(); dtde.dropComplete(false); } if (!tmpFile.delete()) { tmpFile.deleteOnExit(); } } catch (MalformedURLException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } }
From source file:com.flexoodb.common.FlexUtils.java
static public byte[] getFileInBytes(String f) throws Exception { boolean haserror = false; Throwable t = null;/*from w ww .j a va2 s . com*/ byte[] data = null; RandomAccessFile rf = null; try { rf = new RandomAccessFile(f, "r"); int size = (int) rf.length(); data = new byte[size]; rf.readFully(data); } catch (Exception e) { t = e; haserror = true; } finally { if (rf != null) { rf.close(); } if (haserror) { throw new Exception(t); } } return data; }
From source file:org.commoncrawl.service.listcrawler.CrawlHistoryManager.java
/** * /* ww w. j av a 2 s . c om*/ * @return a sorted map of urlfp to item * @throws IOException */ TreeMap<URLFP, ProxyCrawlHistoryItem> loadLocalLogItemMap() throws IOException { TreeMap<URLFP, ProxyCrawlHistoryItem> itemMap = new TreeMap<URLFP, ProxyCrawlHistoryItem>(); LOG.info("Reading Local Log File"); RandomAccessFile file = new RandomAccessFile(getActiveLogFilePath(), "rw"); // valid length indicator ... long validLength = 0; try { // skip header ... file.seek(LocalLogFileHeader.SIZE); validLength = file.getFilePointer(); // ok walk n items ... for (int itemIdx = 0; itemIdx < _header._itemCount && file.getChannel().position() <= _header._fileSize; ++itemIdx) { try { ProxyCrawlHistoryItem item = readItem(file); // update valid length ... validLength = file.getFilePointer(); // ok compute fingerprint for item ... URLFP fingerprintObject = URLUtils.getURLFPFromURL(item.getOriginalURL(), true); if (fingerprintObject == null) { LOG.error("Could not compute fingerprint for URL:" + item.getOriginalURL()); } else { itemMap.put(fingerprintObject, item); } } catch (IOException e) { LOG.error(CCStringUtils.stringifyException(e)); try { if (!seekToNextSyncBytesPos(file)) { LOG.error("Hit EOF While Seeking for next SyncByte Sequence!"); break; } else { LOG.info("Seek to Next SyncByte Succeeded! Continuing Load"); } } catch (IOException e2) { LOG.error(CCStringUtils.stringifyException(e2)); LOG.error("Got IO Exception Reading SyncBytes - Bailing!"); break; } } } } finally { if (file.length() > validLength) { LOG.warn("File Length is:" + file.length() + " Truncating Length to:" + validLength); file.setLength(validLength); } file.close(); } LOG.info("Done Reading Local Log File"); return itemMap; }
From source file:edu.umass.cs.gigapaxos.SQLPaxosLogger.java
private static void compactLogfile(File file, PaxosPacketizer packetizer, MessageLogDiskMap msgLog, FileIDMap fidMap) throws IOException, JSONException { RandomAccessFile raf = null, rafTmp = null; File tmpFile = new File(file.toString() + TMP_FILE_SUFFIX); int tmpFileSize = 0; boolean compacted = false, neededAtAll = false; HashMap<String, ArrayList<LogIndexEntry>> logIndexEntries = new HashMap<String, ArrayList<LogIndexEntry>>(); // quick delete if (fidMap.isRemovable(file.toString(), msgLog)) { deleteFile(file, msgLog);/*from ww w . j a v a 2s . c om*/ log.log(Level.INFO, "{0} quick-garbage-collected file {1}", new Object[] { msgLog.disk, file }); return; } else log.log(Level.FINE, "{0} not quick-GCing file {1} because dependent paxosIDs = {2}", new Object[] { msgLog.disk, file, fidMap.fidMap.get(file.toString()) }); if (System.currentTimeMillis() - file.lastModified() < LOGFILE_AGE_THRESHOLD * 1000) return; try { long t = System.currentTimeMillis(); raf = new RandomAccessFile(file.toString(), "r"); rafTmp = new RandomAccessFile(tmpFile.toString(), "rw"); while (raf.getFilePointer() < raf.length()) { long offset = rafTmp.getFilePointer(); int length = raf.readInt(); byte[] msg = new byte[length]; raf.readFully(msg); PaxosPacket pp = packetizer != null ? packetizer.stringToPaxosPacket(msg // new String(msg, CHARSET) ) : PaxosPacket.getPaxosPacket(new String(msg, CHARSET)); if (!logIndexEntries.containsKey(pp.getPaxosID())) logIndexEntries.put(pp.getPaxosID(), new ArrayList<LogIndexEntry>()); logIndexEntries.get(pp.getPaxosID()).add(new LogIndexEntry(getSlot(pp), getBallot(pp).ballotNumber, getBallot(pp).coordinatorID, pp.getType().getInt(), file.toString(), offset, length)); if (isLogMsgNeeded(pp, msgLog)) { ByteBuffer bbuf = ByteBuffer.allocate(length + 4); bbuf.putInt(length); bbuf.put(msg); rafTmp.write(bbuf.array()); neededAtAll = true; tmpFileSize += bbuf.capacity(); } else { compacted = true; log.log(Level.FINE, "From logfile {0} garbage collecting {1}", new Object[] { file, pp.getSummary() }); } } DelayProfiler.updateDelay("compact", t); } finally { if (raf != null) raf.close(); if (rafTmp != null) { rafTmp.getChannel().force(true); rafTmp.close(); } } assert (tmpFile.exists() && tmpFile.length() == tmpFileSize) : tmpFile.length() + " != " + tmpFileSize; if (compacted && neededAtAll) synchronized (msgLog) { modifyLogfileAndLogIndex(file, tmpFile, logIndexEntries, msgLog, fidMap); log.log(Level.INFO, "{0} compacted logfile {1}", new Object[] { msgLog, file }); } else if (!neededAtAll) { log.log(Level.INFO, "Deleting logfile {0} as its log entries are no longer needed", new Object[] { file }); deleteFile(file, msgLog); } else // !compacted log.log(Level.INFO, "Logfile {0} unchanged upon compaction attempt", new Object[] { file }); assert (tmpFile.toString().endsWith(TMP_FILE_SUFFIX)); if (tmpFile.exists()) deleteFile(tmpFile, msgLog); }
From source file:edu.umass.cs.gigapaxos.SQLPaxosLogger.java
private static void mergeLogfiles(File prev, File cur, PaxosPacketizer packetizer, MessageLogDiskMap msgLog, FileIDMap fidMap) throws IOException, JSONException { File tmpFile = new File(cur.toString() + TMP_FILE_SUFFIX); RandomAccessFile rafTmp = null, rafPrev = null, rafCur = null; long t = System.currentTimeMillis(); try {//from ww w . ja v a 2s. c o m rafTmp = new RandomAccessFile(tmpFile.toString(), "rw"); rafPrev = new RandomAccessFile(prev.toString(), "r"); rafCur = new RandomAccessFile(cur.toString(), "r"); byte[] buf = new byte[1024]; int numRead = 0; // copy prev file to tmp file while ((numRead = rafPrev.read(buf)) > 0) rafTmp.write(buf, 0, numRead); // copy cur file to tmp file while ((numRead = rafCur.read(buf)) > 0) rafTmp.write(buf, 0, numRead); } finally { if (rafTmp != null) rafTmp.close(); if (rafPrev != null) rafPrev.close(); if (rafCur != null) rafCur.close(); } // copy tmp file index into memory HashMap<String, ArrayList<LogIndexEntry>> logIndexEntries = new HashMap<String, ArrayList<LogIndexEntry>>(); try { rafTmp = new RandomAccessFile(tmpFile.toString(), "r"); while (rafTmp.getFilePointer() < rafTmp.length()) { long offset = rafTmp.getFilePointer(); int length = rafTmp.readInt(); byte[] msg = new byte[length]; rafTmp.readFully(msg); PaxosPacket pp = packetizer != null ? packetizer.stringToPaxosPacket(msg // new String(msg, CHARSET) ) : PaxosPacket.getPaxosPacket(new String(msg, CHARSET)); assert (pp != null) : " read logged message " + new String(msg, CHARSET); if (!logIndexEntries.containsKey(pp.getPaxosID())) logIndexEntries.put(pp.getPaxosID(), new ArrayList<LogIndexEntry>()); logIndexEntries.get(pp.getPaxosID()).add(new LogIndexEntry(getSlot(pp), getBallot(pp).ballotNumber, getBallot(pp).coordinatorID, pp.getType().getInt(), cur.toString(), offset, length)); } } finally { if (rafTmp != null) rafTmp.close(); } // atomically copy tmpFile to cur, adjust log index, delete prev synchronized (msgLog) { modifyLogfileAndLogIndex(cur, tmpFile, logIndexEntries, msgLog, fidMap); if (prev.delete()) fidMap.remove(prev.toString()); } DelayProfiler.updateDelay("merge", t); log.log(Level.INFO, "{0} merged logfile {1} into {2}", new Object[] { msgLog, prev, cur }); }
From source file:com.portfolio.rest.RestServicePortfolio.java
@Path("/portfolios/portfolio/{portfolio-id}") @GET/*w w w .j a va2s . c om*/ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, "application/zip", MediaType.APPLICATION_OCTET_STREAM }) public Object getPortfolio(@CookieParam("user") String user, @CookieParam("credential") String token, @QueryParam("group") int groupId, @PathParam("portfolio-id") String portfolioUuid, @Context ServletConfig sc, @Context HttpServletRequest httpServletRequest, @HeaderParam("Accept") String accept, @QueryParam("user") Integer userId, @QueryParam("group") Integer group, @QueryParam("resources") String resource, @QueryParam("files") String files, @QueryParam("export") String export, @QueryParam("lang") String lang) { UserInfo ui = checkCredential(httpServletRequest, user, token, null); Response response = null; try { String portfolio = dataProvider.getPortfolio(new MimeType("text/xml"), portfolioUuid, ui.userId, 0, this.label, resource, "", ui.subId).toString(); if ("faux".equals(portfolio)) { response = Response.status(403).build(); } if (response == null) { Date time = new Date(); Document doc = DomUtils.xmlString2Document(portfolio, new StringBuffer()); NodeList codes = doc.getDocumentElement().getElementsByTagName("code"); // Le premier c'est celui du root Node codenode = codes.item(0); String code = ""; if (codenode != null) code = codenode.getTextContent(); if (export != null) { response = Response.ok(portfolio).header("content-disposition", "attachment; filename = \"" + code + "-" + time + ".xml\"").build(); } else if (resource != null && files != null) { //// Cas du renvoi d'un ZIP /// Temp file in temp directory File tempDir = new File(System.getProperty("java.io.tmpdir", null)); File tempZip = File.createTempFile(portfolioUuid, ".zip", tempDir); FileOutputStream fos = new FileOutputStream(tempZip); ZipOutputStream zos = new ZipOutputStream(fos); // BufferedOutputStream bos = new BufferedOutputStream(zos); /// zos.setComment("Some comment"); /// Write xml file to zip ZipEntry ze = new ZipEntry(portfolioUuid + ".xml"); zos.putNextEntry(ze); byte[] bytes = portfolio.getBytes("UTF-8"); zos.write(bytes); zos.closeEntry(); /// Find all fileid/filename XPath xPath = XPathFactory.newInstance().newXPath(); String filterRes = "//asmResource/fileid"; NodeList nodelist = (NodeList) xPath.compile(filterRes).evaluate(doc, XPathConstants.NODESET); /// Direct link to data // String urlTarget = "http://"+ server + "/user/" + user +"/file/" + uuid +"/"+ lang+ "/ptype/fs"; /* String langatt = ""; if( lang != null ) langatt = "?lang="+lang; else langatt = "?lang=fr"; //*/ /// Fetch all files for (int i = 0; i < nodelist.getLength(); ++i) { Node res = nodelist.item(i); Node p = res.getParentNode(); // resource -> container Node gp = p.getParentNode(); // container -> context Node uuidNode = gp.getAttributes().getNamedItem("id"); String uuid = uuidNode.getTextContent(); String filterName = "./filename[@lang and text()]"; NodeList textList = (NodeList) xPath.compile(filterName).evaluate(p, XPathConstants.NODESET); String filename = ""; if (textList.getLength() != 0) { Element fileNode = (Element) textList.item(0); filename = fileNode.getTextContent(); lang = fileNode.getAttribute("lang"); if ("".equals(lang)) lang = "fr"; } String servlet = httpServletRequest.getRequestURI(); servlet = servlet.substring(0, servlet.indexOf("/", 7)); String server = httpServletRequest.getServerName(); int port = httpServletRequest.getServerPort(); // "http://"+ server + /resources/resource/file/ uuid ? lang= size= // String urlTarget = "http://"+ server + "/user/" + user +"/file/" + uuid +"/"+ lang+ "/ptype/fs"; String url = "http://" + server + ":" + port + servlet + "/resources/resource/file/" + uuid + "?lang=" + lang; HttpGet get = new HttpGet(url); // Transfer sessionid so that local request still get security checked HttpSession session = httpServletRequest.getSession(true); get.addHeader("Cookie", "JSESSIONID=" + session.getId()); // Send request CloseableHttpClient client = HttpClients.createDefault(); CloseableHttpResponse ret = client.execute(get); HttpEntity entity = ret.getEntity(); // Put specific name for later recovery if ("".equals(filename)) continue; int lastDot = filename.lastIndexOf("."); if (lastDot < 0) lastDot = 0; String filenameext = filename.substring(0); /// find extension int extindex = filenameext.lastIndexOf("."); filenameext = uuid + "_" + lang + filenameext.substring(extindex); // Save it to zip file // int length = (int) entity.getContentLength(); InputStream content = entity.getContent(); // BufferedInputStream bis = new BufferedInputStream(entity.getContent()); ze = new ZipEntry(filenameext); try { int totalread = 0; zos.putNextEntry(ze); int inByte; byte[] buf = new byte[4096]; // zos.write(bytes,0,inByte); while ((inByte = content.read(buf)) != -1) { totalread += inByte; zos.write(buf, 0, inByte); } System.out.println("FILE: " + filenameext + " -> " + totalread); content.close(); // bis.close(); zos.closeEntry(); } catch (Exception e) { e.printStackTrace(); } EntityUtils.consume(entity); ret.close(); client.close(); } zos.close(); fos.close(); /// Return zip file RandomAccessFile f = new RandomAccessFile(tempZip.getAbsoluteFile(), "r"); byte[] b = new byte[(int) f.length()]; f.read(b); f.close(); response = Response.ok(b, MediaType.APPLICATION_OCTET_STREAM) .header("content-disposition", "attachment; filename = \"" + code + "-" + time + ".zip") .build(); // Temp file cleanup tempZip.delete(); } else { //try { this.userId = userId; } catch(Exception ex) { this.userId = -1; }; // String returnValue = dataProvider.getPortfolio(new MimeType("text/xml"),portfolioUuid,this.userId, this.groupId, this.label, resource, files).toString(); if (portfolio.equals("faux")) { throw new RestWebApplicationException(Status.FORBIDDEN, "Vous n'avez pas les droits necessaires"); } if (accept.equals(MediaType.APPLICATION_JSON)) { portfolio = XML.toJSONObject(portfolio).toString(); response = Response.ok(portfolio).type(MediaType.APPLICATION_JSON).build(); } else response = Response.ok(portfolio).type(MediaType.APPLICATION_XML).build(); logRestRequest(httpServletRequest, null, portfolio, Status.OK.getStatusCode()); } } } catch (RestWebApplicationException ex) { throw new RestWebApplicationException(Status.FORBIDDEN, ex.getResponse().getEntity().toString()); } catch (SQLException ex) { logRestRequest(httpServletRequest, null, "Portfolio " + portfolioUuid + " not found", Status.NOT_FOUND.getStatusCode()); throw new RestWebApplicationException(Status.NOT_FOUND, "Portfolio " + portfolioUuid + " not found"); } catch (Exception ex) { ex.printStackTrace(); logRestRequest(httpServletRequest, null, ex.getMessage() + "\n\n" + ex.getStackTrace(), Status.INTERNAL_SERVER_ERROR.getStatusCode()); throw new RestWebApplicationException(Status.INTERNAL_SERVER_ERROR, ex.getMessage()); } finally { if (dataProvider != null) dataProvider.disconnect(); } return response; }
From source file:edu.umass.cs.gigapaxos.SQLPaxosLogger.java
private byte[] getJournaledMessage(String logfile, long offset, int length, RandomAccessFile raf) throws IOException { assert (logfile != null); if (!new File(logfile).exists()) return null; boolean locallyOpened = false; if (raf == null) { locallyOpened = true;//from w ww . j av a 2s .c o m raf = new RandomAccessFile(logfile, "r"); } boolean error = false; String msg = null; byte[] buf = null; try { raf.seek(offset); assert (raf.length() > offset) : this + " " + raf.length() + " <= " + offset + " while reading logfile " + logfile; int readLength = raf.readInt(); try { assert (readLength == length) : this + " : " + readLength + " != " + length; } catch (Error e) { error = true; log.severe(this + ": " + e); e.printStackTrace(); } int bufLength = length; buf = new byte[bufLength]; raf.readFully(buf); if (JOURNAL_COMPRESSION) buf = inflate(buf); msg = new String(buf, CHARSET); } catch (IOException | Error e) { log.log(Level.INFO, "{0} incurred IOException while retrieving journaled message {1}:{2}", new Object[] { this, logfile, offset + ":" + length }); e.printStackTrace(); if (locallyOpened) raf.close(); throw e; } log.log(error ? Level.INFO : Level.FINEST, "{0} returning journaled message from {1}:{2} = [{3}]", new Object[] { this, logfile, offset + ":" + length, msg }); return buf;// msg; }