List of usage examples for java.io DataInputStream readInt
public final int readInt() throws IOException
readInt
method of DataInput
. From source file:org.commoncrawl.service.listcrawler.CacheManager.java
/** * loadCacheItemFromDisk - load a single cache item from disk * //from w w w .j ava2s .c o m * @param file * @param optTargetURL * @param location * @return * @throws IOException */ private CacheItem loadCacheItemFromDisk(FileInputStream file, String optTargetURL, long location) throws IOException { long timeStart = System.currentTimeMillis(); // and read out the Item Header ... CacheItemHeader itemHeader = new CacheItemHeader(); itemHeader.readHeader(new DataInputStream(file)); // see if it is valid ... if (!Arrays.equals(itemHeader._sync, _header._sync)) { LOG.error("### Item Lookup for URL:" + optTargetURL + " Record at:" + location + " failed - corrupt sync bytes detected!!!"); } else { CRC32 crc32 = new CRC32(); // ok deserialize the bytes ... CacheItem item = new CacheItem(); CheckedInputStream checkedStream = new CheckedInputStream(file, crc32); DataInputStream itemStream = new DataInputStream(checkedStream); item.readFields(itemStream); // read the content buffer length int contentBufferLen = itemStream.readInt(); if (contentBufferLen != 0) { byte data[] = new byte[contentBufferLen]; itemStream.read(data); item.setContent(new Buffer(data)); } // cache crc long crcValueComputed = crc32.getValue(); // read disk crc long crcValueOnDisk = itemStream.readLong(); // validate if (crcValueComputed == crcValueOnDisk) { String canonicalURL = URLUtils.canonicalizeURL(item.getUrl(), true); if (optTargetURL.length() == 0 || optTargetURL.equals(canonicalURL)) { if (isValidCacheItem(item)) { LOG.info("### Item Lookup for URL:" + optTargetURL + " Record at:" + location + " completed in:" + (System.currentTimeMillis() - timeStart)); return item; } else { LOG.info("### Item Lookup for URL:" + optTargetURL + " Record at:" + location + " failed with invalid result code"); } } else { LOG.info("### Item Lookup for URL:" + optTargetURL + " Record at:" + location + " failed with url mismatch. record url:" + item.getUrl()); } } else { LOG.error("### Item Lookup for URL:" + optTargetURL + " Record at:" + location + " failed - crc mismatch!!!"); } } return null; }
From source file:org.apache.giraph.worker.BspServiceSource.java
@Override public VertexEdgeCount loadCheckpoint(long superstep) { Path metadataFilePath = getSavedCheckpoint(superstep, CheckpointingUtils.CHECKPOINT_METADATA_POSTFIX); Path checkpointFilePath = getSavedCheckpoint(superstep, CheckpointingUtils.CHECKPOINT_DATA_POSTFIX); // Algorithm: // Examine all the partition owners and load the ones // that match my hostname and id from the master designated checkpoint // prefixes./*from w ww . j av a 2s. co m*/ try { DataInputStream metadataStream = getFs().open(metadataFilePath); int partitions = metadataStream.readInt(); List<Integer> partitionIds = new ArrayList<>(partitions); for (int i = 0; i < partitions; i++) { int partitionId = metadataStream.readInt(); partitionIds.add(partitionId); } loadCheckpointVertices(superstep, partitionIds); getContext().progress(); metadataStream.close(); DataInputStream checkpointStream = getFs().open(checkpointFilePath); workerContext.readFields(checkpointStream); // Load global stats and superstep classes GlobalStats globalStats = new GlobalStats(); SuperstepClasses superstepClasses = new SuperstepClasses(); String finalizedCheckpointPath = getSavedCheckpointBasePath(superstep) + CheckpointingUtils.CHECKPOINT_FINALIZED_POSTFIX; DataInputStream finalizedStream = getFs().open(new Path(finalizedCheckpointPath)); globalStats.readFields(finalizedStream); superstepClasses.readFields(finalizedStream); getConfiguration().updateSuperstepClasses(superstepClasses); getServerData().resetMessageStores(); for (int i = 0; i < partitions; i++) { int partitionId = checkpointStream.readInt(); getServerData().getCurrentMessageStore().readFieldsForPartition(checkpointStream, partitionId); } List<Writable> w2wMessages = (List<Writable>) WritableUtils.readList(checkpointStream); getServerData().getCurrentWorkerToWorkerMessages().addAll(w2wMessages); checkpointStream.close(); if (LOG.isInfoEnabled()) { LOG.info( "loadCheckpoint: Loaded " + workerGraphPartitioner.getPartitionOwners().size() + " total."); } // Communication service needs to setup the connections prior to // processing vertices workerClient.setup(getConfiguration().authenticate()); return new VertexEdgeCount(globalStats.getVertexCount(), globalStats.getEdgeCount()); } catch (IOException e) { throw new RuntimeException("loadCheckpoint: Failed for superstep=" + superstep, e); } }
From source file:org.apache.geode.internal.cache.tier.sockets.HandShake.java
/** * HandShake Constructor used by server side connection *//* www . j ava2s . co m*/ public HandShake(Socket sock, int timeout, DistributedSystem sys, Version clientVersion, CommunicationMode communicationMode, SecurityService securityService) throws IOException, AuthenticationRequiredException { this.clientVersion = clientVersion; this.system = sys; this.securityService = securityService; { int soTimeout = -1; try { soTimeout = sock.getSoTimeout(); sock.setSoTimeout(timeout); InputStream is = sock.getInputStream(); int valRead = is.read(); // this.code = (byte)is.read(); if (valRead == -1) { throw new EOFException( LocalizedStrings.HandShake_HANDSHAKE_EOF_REACHED_BEFORE_CLIENT_CODE_COULD_BE_READ .toLocalizedString()); } this.code = (byte) valRead; if (this.code != REPLY_OK) { throw new IOException( LocalizedStrings.HandShake_HANDSHAKE_REPLY_CODE_IS_NOT_OK.toLocalizedString()); } try { DataInputStream dis = new DataInputStream(is); DataOutputStream dos = new DataOutputStream(sock.getOutputStream()); this.clientReadTimeout = dis.readInt(); if (clientVersion.compareTo(Version.CURRENT) < 0) { // versioned streams allow object serialization code to deal with older clients dis = new VersionedDataInputStream(dis, clientVersion); dos = new VersionedDataOutputStream(dos, clientVersion); } this.id = ClientProxyMembershipID.readCanonicalized(dis); // Note: credentials should always be the last piece in handshake for // Diffie-Hellman key exchange to work if (clientVersion.compareTo(Version.GFE_603) >= 0) { setOverrides(new byte[] { dis.readByte() }); } else { setClientConflation(dis.readByte()); } // Hitesh if (this.clientVersion.compareTo(Version.GFE_65) < 0 || communicationMode.isWAN()) { this.credentials = readCredentials(dis, dos, sys, this.securityService); } else { this.credentials = this.readCredential(dis, dos, sys); } } catch (IOException ioe) { this.code = -2; throw ioe; } catch (ClassNotFoundException cnfe) { this.code = -3; throw new IOException( LocalizedStrings.HandShake_CLIENTPROXYMEMBERSHIPID_CLASS_COULD_NOT_BE_FOUND_WHILE_DESERIALIZING_THE_OBJECT .toLocalizedString()); } } finally { if (soTimeout != -1) { try { sock.setSoTimeout(soTimeout); } catch (IOException ignore) { } } } } }
From source file:voldemort.server.protocol.admin.AdminServiceRequestHandler.java
/** * This method is used by non-blocking code to determine if the give buffer * represents a complete request. Because the non-blocking code can by * definition not just block waiting for more data, it's possible to get * partial reads, and this identifies that case. * * @param buffer Buffer to check; the buffer is reset to position 0 before * calling this method and the caller must reset it after the call * returns//from w ww. j a va 2 s . com * @return True if the buffer holds a complete request, false otherwise */ @Override public boolean isCompleteRequest(ByteBuffer buffer) { DataInputStream inputStream = new DataInputStream(new ByteBufferBackedInputStream(buffer)); try { int dataSize = inputStream.readInt(); if (logger.isTraceEnabled()) logger.trace( "In isCompleteRequest, dataSize: " + dataSize + ", buffer position: " + buffer.position()); if (dataSize == -1) return true; // Here we skip over the data (without reading it in) and // move our position to just past it. buffer.position(buffer.position() + dataSize); return true; } catch (Exception e) { // This could also occur if the various methods we call into // re-throw a corrupted value error as some other type of exception. // For example, updating the position on a buffer past its limit // throws an InvalidArgumentException. if (logger.isTraceEnabled()) logger.trace("In isCompleteRequest, probable partial read occurred: " + e); return false; } }
From source file:org.apache.giraph.master.BspServiceMaster.java
/** * Read the finalized checkpoint file and associated metadata files for the * checkpoint. Modifies the {@link PartitionOwner} objects to get the * checkpoint prefixes. It is an optimization to prevent all workers from * searching all the files. Also read in the aggregator data from the * finalized checkpoint file and setting it. * * @param superstep Checkpoint set to examine. * @throws IOException/*from w w w.ja va2 s.c o m*/ * @throws InterruptedException * @throws KeeperException * @return Collection of generated partition owners. */ private Collection<PartitionOwner> prepareCheckpointRestart(long superstep) throws IOException, KeeperException, InterruptedException { List<PartitionOwner> partitionOwners = new ArrayList<>(); FileSystem fs = getFs(); String finalizedCheckpointPath = getSavedCheckpointBasePath(superstep) + CheckpointingUtils.CHECKPOINT_FINALIZED_POSTFIX; LOG.info("Loading checkpoint from " + finalizedCheckpointPath); DataInputStream finalizedStream = fs.open(new Path(finalizedCheckpointPath)); GlobalStats globalStats = new GlobalStats(); globalStats.readFields(finalizedStream); updateCounters(globalStats); SuperstepClasses superstepClasses = SuperstepClasses.createToRead(getConfiguration()); superstepClasses.readFields(finalizedStream); getConfiguration().updateSuperstepClasses(superstepClasses); int prefixFileCount = finalizedStream.readInt(); String checkpointFile = finalizedStream.readUTF(); for (int i = 0; i < prefixFileCount; ++i) { int mrTaskId = finalizedStream.readInt(); DataInputStream metadataStream = fs.open( new Path(checkpointFile + "." + mrTaskId + CheckpointingUtils.CHECKPOINT_METADATA_POSTFIX)); long partitions = metadataStream.readInt(); WorkerInfo worker = getWorkerInfoById(mrTaskId); for (long p = 0; p < partitions; ++p) { int partitionId = metadataStream.readInt(); PartitionOwner partitionOwner = new BasicPartitionOwner(partitionId, worker); partitionOwners.add(partitionOwner); LOG.info("prepareCheckpointRestart partitionId=" + partitionId + " assigned to " + partitionOwner); } metadataStream.close(); } //Ordering appears to be important as of right now we rely on this ordering //in WorkerGraphPartitioner Collections.sort(partitionOwners, new Comparator<PartitionOwner>() { @Override public int compare(PartitionOwner p1, PartitionOwner p2) { return Integer.compare(p1.getPartitionId(), p2.getPartitionId()); } }); globalCommHandler.getAggregatorHandler().readFields(finalizedStream); aggregatorTranslation.readFields(finalizedStream); masterCompute.readFields(finalizedStream); finalizedStream.close(); return partitionOwners; }
From source file:com.android.leanlauncher.LauncherTransitionable.java
private static void readConfiguration(Context context, LocaleConfiguration configuration) { DataInputStream in = null; try {/* w ww . j ava 2 s . co m*/ in = new DataInputStream(context.openFileInput(LauncherFiles.LAUNCHER_PREFERENCES)); configuration.locale = in.readUTF(); configuration.mcc = in.readInt(); configuration.mnc = in.readInt(); } catch (FileNotFoundException e) { // Ignore } catch (IOException e) { // Ignore } finally { if (in != null) { try { in.close(); } catch (IOException e) { // Ignore } } } }
From source file:it.unimi.di.big.mg4j.document.SimpleCompressedDocumentCollection.java
public Document document(long index) throws IOException { ensureDocumentIndex(index);//from w ww . j a va 2s .com ensureFiles(); documentsInputBitStream.position(docOffsets.getLong(index)); @SuppressWarnings("resource") final DataInputStream nonTextDataInputStream = hasNonText ? new DataInputStream( new FastBufferedInputStream(zipFile.getInputStream(zipFile.getEntry(Long.toString(index))))) : null; final MutableString uri = readSelfDelimitedUtf8String(documentsInputBitStream, new MutableString()); final MutableString title = readSelfDelimitedUtf8String(documentsInputBitStream, new MutableString()); return new AbstractDocument() { final MutableString fieldContent = new MutableString(); @SuppressWarnings("unchecked") final Document fakeDocument = factory.getDocument(NullInputStream.getInstance(), Reference2ObjectMaps.EMPTY_MAP); int nextField = 0; public Object content(int field) throws IOException { FieldType fieldType = factory.fieldType(field); if (nextField > field) throw new IllegalStateException(); // Skip fields final MutableString s = new MutableString(); int len; while (nextField < field) { switch (factory.fieldType(nextField)) { case TEXT: len = documentsInputBitStream.readDelta(); if (exact) len *= 2; documentsInputBitStream.skipDeltas(len); break; case VIRTUAL: final int nfrag = nonTextDataInputStream.readInt(); for (int i = 0; i < 2 * nfrag; i++) MutableString.skipSelfDelimUTF8(nonTextDataInputStream); break; default: try { new ObjectInputStream(nonTextDataInputStream).readObject(); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } nextField++; } // Read field nextField++; switch (fieldType) { case TEXT: len = documentsInputBitStream.readDelta(); fieldContent.length(0); termsFrequencyKeeper.reset(); if (exact) nonTermsFrequencyKeeper.reset(); while (len-- != 0) { termsInputStream.position(termOffsets .getLong(termsFrequencyKeeper.decode(documentsInputBitStream.readDelta()))); s.readSelfDelimUTF8(termsInputStream); fieldContent.append(s); if (exact) { nonTermsInputStream.position(nonTermOffsets .getLong(nonTermsFrequencyKeeper.decode(documentsInputBitStream.readDelta()))); s.readSelfDelimUTF8(nonTermsInputStream); fieldContent.append(s); } else fieldContent.append(' '); } return new FastBufferedReader(fieldContent); case VIRTUAL: final int nfrag = nonTextDataInputStream.readInt(); MutableString doc = new MutableString(); MutableString text = new MutableString(); VirtualDocumentFragment[] fragArray = new VirtualDocumentFragment[nfrag]; for (int i = 0; i < nfrag; i++) { doc.readSelfDelimUTF8((InputStream) nonTextDataInputStream); text.readSelfDelimUTF8((InputStream) nonTextDataInputStream); fragArray[i] = new AnchorExtractor.Anchor(doc.copy(), text.copy()); } return new ObjectArrayList<VirtualDocumentFragment>(fragArray); default: try { return new ObjectInputStream(nonTextDataInputStream).readObject(); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } } public CharSequence title() { return title; } public CharSequence uri() { return uri.length() == 0 ? null : uri; } public WordReader wordReader(int field) { switch (factory.fieldType(field)) { case TEXT: case VIRTUAL: return fakeDocument.wordReader(field); default: return null; } } public void close() throws IOException { super.close(); if (hasNonText) nonTextDataInputStream.close(); } }; }
From source file:voldemort.server.protocol.admin.AdminServiceRequestHandler.java
@Override public StreamRequestHandler handleRequest(final DataInputStream inputStream, final DataOutputStream outputStream, final ByteBufferContainer container) throws IOException { // Another protocol buffers bug here, temp. work around VoldemortAdminRequest.Builder request = VoldemortAdminRequest.newBuilder(); int size = inputStream.readInt(); if (logger.isTraceEnabled()) logger.trace("In handleRequest, request specified size of " + size + " bytes"); if (size < 0) throw new IOException("In handleRequest, request specified size of " + size + " bytes"); byte[] input = new byte[size]; ByteUtils.read(inputStream, input);/*w w w . j a v a 2 s . c om*/ request.mergeFrom(input); switch (request.getType()) { case GET_METADATA: ProtoUtils.writeMessage(outputStream, handleGetMetadata(request.getGetMetadata())); break; case UPDATE_METADATA: ProtoUtils.writeMessage(outputStream, handleSetMetadata(request.getUpdateMetadata())); break; case UPDATE_STORE_DEFINITIONS: ProtoUtils.writeMessage(outputStream, handleUpdateStoreDefinitions(request.getUpdateMetadata())); break; case UPDATE_METADATA_PAIR: ProtoUtils.writeMessage(outputStream, handleUpdateMetadataPair(request.getUpdateMetadataPair())); break; case DELETE_PARTITION_ENTRIES: ProtoUtils.writeMessage(outputStream, handleDeletePartitionEntries(request.getDeletePartitionEntries())); break; case FETCH_PARTITION_ENTRIES: return handleFetchPartitionEntries(request.getFetchPartitionEntries()); case UPDATE_PARTITION_ENTRIES: return handleUpdatePartitionEntries(request.getUpdatePartitionEntries()); case INITIATE_FETCH_AND_UPDATE: ProtoUtils.writeMessage(outputStream, handleFetchAndUpdate(request.getInitiateFetchAndUpdate())); break; case ASYNC_OPERATION_STATUS: ProtoUtils.writeMessage(outputStream, handleAsyncStatus(request.getAsyncOperationStatus())); break; case INITIATE_REBALANCE_NODE: ProtoUtils.writeMessage(outputStream, handleRebalanceNode(request.getInitiateRebalanceNode())); break; case ASYNC_OPERATION_LIST: ProtoUtils.writeMessage(outputStream, handleAsyncOperationList(request.getAsyncOperationList())); break; case ASYNC_OPERATION_STOP: ProtoUtils.writeMessage(outputStream, handleAsyncOperationStop(request.getAsyncOperationStop())); break; case LIST_SCHEDULED_JOBS: ProtoUtils.writeMessage(outputStream, handleListScheduledJobs(request.getListScheduledJobs())); break; case GET_SCHEDULED_JOB_STATUS: ProtoUtils.writeMessage(outputStream, handleGetScheduledJobStatus(request.getGetScheduledJobStatus())); break; case STOP_SCHEDULED_JOB: ProtoUtils.writeMessage(outputStream, handleStopScheduledJob(request.getStopScheduledJob())); break; case ENABLE_SCHEDULED_JOB: ProtoUtils.writeMessage(outputStream, handleEnableScheduledJob(request.getEnableScheduledJob())); break; case TRUNCATE_ENTRIES: ProtoUtils.writeMessage(outputStream, handleTruncateEntries(request.getTruncateEntries())); break; case ADD_STORE: ProtoUtils.writeMessage(outputStream, handleAddStore(request.getAddStore())); break; case DELETE_STORE: ProtoUtils.writeMessage(outputStream, handleDeleteStore(request.getDeleteStore())); break; case FETCH_STORE: ProtoUtils.writeMessage(outputStream, handleFetchROStore(request.getFetchStore())); break; case SWAP_STORE: ProtoUtils.writeMessage(outputStream, handleSwapROStore(request.getSwapStore())); break; case ROLLBACK_STORE: ProtoUtils.writeMessage(outputStream, handleRollbackStore(request.getRollbackStore())); break; case GET_RO_MAX_VERSION_DIR: ProtoUtils.writeMessage(outputStream, handleGetROMaxVersionDir(request.getGetRoMaxVersionDir())); break; case GET_RO_CURRENT_VERSION_DIR: ProtoUtils.writeMessage(outputStream, handleGetROCurrentVersionDir(request.getGetRoCurrentVersionDir())); break; case GET_RO_STORAGE_FORMAT: ProtoUtils.writeMessage(outputStream, handleGetROStorageFormat(request.getGetRoStorageFormat())); break; case GET_RO_STORAGE_FILE_LIST: ProtoUtils.writeMessage(outputStream, handleGetROStorageFileList(request.getGetRoStorageFileList())); break; case GET_RO_COMPRESSION_CODEC_LIST: ProtoUtils.writeMessage(outputStream, handleGetROCompressionCodecList(request.getGetRoCompressionCodecList())); break; case FETCH_PARTITION_FILES: return handleFetchROPartitionFiles(request.getFetchPartitionFiles()); case UPDATE_SLOP_ENTRIES: return handleUpdateSlopEntries(request.getUpdateSlopEntries()); case FAILED_FETCH_STORE: ProtoUtils.writeMessage(outputStream, handleFailedROFetch(request.getFailedFetchStore())); break; case REBALANCE_STATE_CHANGE: ProtoUtils.writeMessage(outputStream, handleRebalanceStateChange(request.getRebalanceStateChange())); break; case DELETE_STORE_REBALANCE_STATE: ProtoUtils.writeMessage(outputStream, handleDeleteStoreRebalanceState(request.getDeleteStoreRebalanceState())); break; case SET_OFFLINE_STATE: ProtoUtils.writeMessage(outputStream, handleSetOfflineState(request.getSetOfflineState())); break; case REPAIR_JOB: ProtoUtils.writeMessage(outputStream, handleRepairJob(request.getRepairJob())); break; case PRUNE_JOB: ProtoUtils.writeMessage(outputStream, handlePruneJob(request.getPruneJob())); break; case SLOP_PURGE_JOB: ProtoUtils.writeMessage(outputStream, handleSlopPurgeJob(request.getSlopPurgeJob())); break; case NATIVE_BACKUP: ProtoUtils.writeMessage(outputStream, handleNativeBackup(request.getNativeBackup())); break; case RESERVE_MEMORY: ProtoUtils.writeMessage(outputStream, handleReserveMemory(request.getReserveMemory())); break; case GET_HA_SETTINGS: ProtoUtils.writeMessage(outputStream, handleGetHighAvailabilitySettings(request.getGetHaSettings())); break; case DISABLE_STORE_VERSION: ProtoUtils.writeMessage(outputStream, handleDisableStoreVersion(request.getDisableStoreVersion())); break; case HANDLE_FETCH_FAILURE: ProtoUtils.writeMessage(outputStream, handleFetchFailure(request.getHandleFetchFailure())); break; case GET_CONFIG: ProtoUtils.writeMessage(outputStream, handleGetConfigRequest(request.getGetConfig())); break; default: throw new VoldemortException("Unknown operation: " + request.getType()); } return null; }
From source file:mp.teardrop.PlaybackService.java
/** * Initializes the service state, loading songs saved from the disk into the * song timeline.// w w w. j a va 2s .c o m * * @return The loaded value for mState. */ public int loadState() { int state = 0; try { DataInputStream in = new DataInputStream(openFileInput(STATE_FILE)); if (in.readLong() == STATE_FILE_MAGIC && in.readInt() == STATE_VERSION) { mPendingSeek = in.readInt(); mPendingSeekSong = in.readLong(); mTimeline.readState(getSharedPreferences(PREFS_SAVED_SONGS, 0)); state |= mTimeline.getShuffleMode() << SHIFT_SHUFFLE; state |= mTimeline.getFinishAction() << SHIFT_FINISH; } in.close(); } catch (EOFException e) { Log.w("OrchidMP", "Failed to load state", e); } catch (IOException e) { Log.w("OrchidMP", "Failed to load state", e); } catch (JSONException e) { Log.w("OrchidMP", "Failed to load state", e); } return state; }