List of usage examples for java.lang Thread interrupted
public static boolean interrupted()
From source file:edu.mit.mobile.android.locast.sync.SyncEngine.java
/** * @param toSync/* www.jav a2 s . c o m*/ * @param account * @param extras * @param provider * @param syncResult * @return true if the item was sync'd successfully. Soft errors will cause this to return * false. * @throws RemoteException * @throws SyncException * @throws JSONException * @throws IOException * @throws NetworkProtocolException * @throws NoPublicPath * @throws OperationApplicationException * @throws InterruptedException */ public boolean sync(Uri toSync, Account account, Bundle extras, ContentProviderClient provider, SyncResult syncResult) throws RemoteException, SyncException, JSONException, IOException, NetworkProtocolException, NoPublicPath, OperationApplicationException, InterruptedException { String pubPath = null; // // Handle http or https uris separately. These require the // destination uri. // if ("http".equals(toSync.getScheme()) || "https".equals(toSync.getScheme())) { pubPath = toSync.toString(); if (!extras.containsKey(EXTRA_DESTINATION_URI)) { throw new IllegalArgumentException("missing EXTRA_DESTINATION_URI when syncing HTTP URIs"); } toSync = Uri.parse(extras.getString(EXTRA_DESTINATION_URI)); } final String type = provider.getType(toSync); final boolean isDir = type.startsWith(CONTENT_TYPE_PREFIX_DIR); final boolean manualSync = extras.getBoolean(ContentResolver.SYNC_EXTRAS_MANUAL, false); // skip any items already sync'd if (!manualSync && mLastUpdated.isUpdatedRecently(toSync)) { if (DEBUG) { Log.d(TAG, "not syncing " + toSync + " as it's been updated recently"); } syncResult.stats.numSkippedEntries++; return false; } // the sync map will convert the json data to ContentValues final SyncMap syncMap = MediaProvider.getSyncMap(provider, toSync); final Uri toSyncWithoutQuerystring = toSync.buildUpon().query(null).build(); final HashMap<String, SyncStatus> syncStatuses = new HashMap<String, SyncEngine.SyncStatus>(); final ArrayList<ContentProviderOperation> cpo = new ArrayList<ContentProviderOperation>(); final LinkedList<String> cpoPubUris = new LinkedList<String>(); // // first things first, upload any content that needs to be // uploaded. // try { uploadUnpublished(toSync, account, provider, syncMap, syncStatuses, syncResult); if (Thread.interrupted()) { throw new InterruptedException(); } // this should ensure that all items have a pubPath when we // query it below. if (pubPath == null) { // we should avoid calling this too much as it // can be expensive pubPath = MediaProvider.getPublicPath(mContext, toSync); } } catch (final NoPublicPath e) { // TODO this is a special case and this is probably not the best place to handle this. // Ideally, this should be done in such a way as to reduce any extra DB queries - // perhaps by doing a join with the parent. if (syncMap.isFlagSet(SyncMap.FLAG_PARENT_MUST_SYNC_FIRST)) { if (DEBUG) { Log.d(TAG, "skipping " + toSync + " whose parent hasn't been sync'd first"); } syncResult.stats.numSkippedEntries++; return false; } // if it's an item, we can handle it. if (isDir) { throw e; } } if (pubPath == null) { // this should have been updated already by the initial // upload, so something must be wrong throw new SyncException("never got a public path for " + toSync); } if (DEBUG) { Log.d(TAG, "sync(toSync=" + toSync + ", account=" + account + ", extras=" + extras + ", manualSync=" + manualSync + ",...)"); Log.d(TAG, "pubPath: " + pubPath); } final long request_time = System.currentTimeMillis(); HttpResponse hr = mNetworkClient.get(pubPath); final long response_time = System.currentTimeMillis(); // the time compensation below allows a time-based synchronization to function even if the // local clock is entirely wrong. The server's time is extracted using the Date header and // all are compared relative to the respective clock reference. Any data that's stored on // the mobile should be stored relative to the local clock and the server will respect the // same. long serverTime; try { serverTime = getServerTime(hr); } catch (final DateParseException e) { Log.w(TAG, "could not retrieve date from server. Using local time, which may be incorrect.", e); serverTime = System.currentTimeMillis(); } // TODO check out // http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html final long response_delay = response_time - request_time; if (DEBUG) { Log.d(TAG, "request took " + response_delay + "ms"); } final long localTime = request_time; // add this to the server time to get the local time final long localOffset = (localTime - serverTime); if (Math.abs(localOffset) > 30 * 60 * 1000) { Log.w(TAG, "local clock is off by " + localOffset + "ms"); } if (Thread.interrupted()) { throw new InterruptedException(); } final HttpEntity ent = hr.getEntity(); String selection; String selectionInverse; String[] selectionArgs; if (isDir) { final JSONArray ja = new JSONArray(StreamUtils.inputStreamToString(ent.getContent())); ent.consumeContent(); final int len = ja.length(); selectionArgs = new String[len]; // build the query to see which items are already in the // database final StringBuilder sb = new StringBuilder(); sb.append("("); for (int i = 0; i < len; i++) { if (Thread.interrupted()) { throw new InterruptedException(); } final SyncStatus syncStatus = loadItemFromJsonObject(ja.getJSONObject(i), syncMap, serverTime); syncStatuses.put(syncStatus.remote, syncStatus); selectionArgs[i] = syncStatus.remote; // add in a placeholder for the query sb.append('?'); if (i != (len - 1)) { sb.append(','); } } sb.append(")"); final String placeholders = sb.toString(); selection = JsonSyncableItem._PUBLIC_URI + " IN " + placeholders; selectionInverse = JsonSyncableItem._PUBLIC_URI + " NOT IN " + placeholders; } else { final JSONObject jo = new JSONObject(StreamUtils.inputStreamToString(ent.getContent())); ent.consumeContent(); final SyncStatus syncStatus = loadItemFromJsonObject(jo, syncMap, serverTime); syncStatuses.put(syncStatus.remote, syncStatus); selection = JsonSyncableItem._PUBLIC_URI + "=?"; selectionInverse = JsonSyncableItem._PUBLIC_URI + "!=?"; selectionArgs = new String[] { syncStatus.remote }; } // first check without the querystring. This will ensure that we // properly mark things that we already have in the database. final Cursor check = provider.query(toSyncWithoutQuerystring, SYNC_PROJECTION, selection, selectionArgs, null); // these items are on both sides try { final int pubUriCol = check.getColumnIndex(JsonSyncableItem._PUBLIC_URI); final int idCol = check.getColumnIndex(JsonSyncableItem._ID); // All the items in this cursor should be found on both // the client and the server. for (check.moveToFirst(); !check.isAfterLast(); check.moveToNext()) { if (Thread.interrupted()) { throw new InterruptedException(); } final long id = check.getLong(idCol); final Uri localUri = ContentUris.withAppendedId(toSync, id); final String pubUri = check.getString(pubUriCol); final SyncStatus itemStatus = syncStatuses.get(pubUri); itemStatus.state = SyncState.BOTH_UNKNOWN; itemStatus.local = localUri; // make the status searchable by both remote and // local uri syncStatuses.put(localUri.toString(), itemStatus); } } finally { check.close(); } Cursor c = provider.query(toSync, SYNC_PROJECTION, selection, selectionArgs, null); // these items are on both sides try { final int pubUriCol = c.getColumnIndex(JsonSyncableItem._PUBLIC_URI); final int localModifiedCol = c.getColumnIndex(JsonSyncableItem._MODIFIED_DATE); final int serverModifiedCol = c.getColumnIndex(JsonSyncableItem._SERVER_MODIFIED_DATE); final int idCol = c.getColumnIndex(JsonSyncableItem._ID); // All the items in this cursor should be found on both // the client and the server. for (c.moveToFirst(); !c.isAfterLast(); c.moveToNext()) { if (Thread.interrupted()) { throw new InterruptedException(); } final long id = c.getLong(idCol); final Uri localUri = ContentUris.withAppendedId(toSync, id); final String pubUri = c.getString(pubUriCol); final SyncStatus itemStatus = syncStatuses.get(pubUri); if (itemStatus.state == SyncState.ALREADY_UP_TO_DATE || itemStatus.state == SyncState.NOW_UP_TO_DATE) { if (DEBUG) { Log.d(TAG, localUri + "(" + pubUri + ")" + " is already up to date."); } continue; } itemStatus.local = localUri; // make the status searchable by both remote and local uri syncStatuses.put(localUri.toString(), itemStatus); // last modified as stored in the DB, in phone time final long itemLocalModified = c.getLong(localModifiedCol); // last modified as stored in the DB, in server time final long itemServerModified = c.getLong(serverModifiedCol); final long localAge = localTime - itemLocalModified; final long remoteAge = serverTime - itemStatus.remoteModifiedTime; final long ageDifference = Math.abs(localAge - remoteAge); // up to date, as far remote -> local goes if (itemServerModified == itemStatus.remoteModifiedTime) { itemStatus.state = SyncState.ALREADY_UP_TO_DATE; if (DEBUG) { Log.d(TAG, pubUri + " is up to date."); } // need to download } else if (localAge > remoteAge) { if (DEBUG) { final long serverModified = itemStatus.remoteModifiedTime; Log.d(TAG, pubUri + " : local is " + ageDifference + "ms older (" + android.text.format.DateUtils.formatDateTime(mContext, itemLocalModified, FORMAT_ARGS_DEBUG) + ") than remote (" + android.text.format.DateUtils.formatDateTime(mContext, serverModified, FORMAT_ARGS_DEBUG) + "); updating local copy..."); } itemStatus.state = SyncState.REMOTE_DIRTY; final ContentProviderOperation.Builder b = ContentProviderOperation.newUpdate(localUri); // update this so it's in the local timescale correctServerOffset(itemStatus.remoteCVs, JsonSyncableItem._CREATED_DATE, JsonSyncableItem._CREATED_DATE, localOffset); correctServerOffset(itemStatus.remoteCVs, JsonSyncableItem._SERVER_MODIFIED_DATE, JsonSyncableItem._MODIFIED_DATE, localOffset); b.withValues(itemStatus.remoteCVs); b.withExpectedCount(1); cpo.add(b.build()); cpoPubUris.add(pubUri); syncResult.stats.numUpdates++; // need to upload } else if (localAge < remoteAge) { if (DEBUG) { final long serverModified = itemStatus.remoteModifiedTime; Log.d(TAG, pubUri + " : local is " + ageDifference + "ms newer (" + android.text.format.DateUtils.formatDateTime(mContext, itemLocalModified, FORMAT_ARGS_DEBUG) + ") than remote (" + android.text.format.DateUtils.formatDateTime(mContext, serverModified, FORMAT_ARGS_DEBUG) + "); publishing to server..."); } itemStatus.state = SyncState.LOCAL_DIRTY; mNetworkClient.putJson(pubPath, JsonSyncableItem.toJSON(mContext, localUri, c, syncMap)); } mLastUpdated.markUpdated(localUri); syncResult.stats.numEntries++; } // end for } finally { c.close(); } /* * Apply updates in bulk */ if (cpo.size() > 0) { if (DEBUG) { Log.d(TAG, "applying " + cpo.size() + " bulk updates..."); } final ContentProviderResult[] r = provider.applyBatch(cpo); if (DEBUG) { Log.d(TAG, "Done applying updates. Running postSync handler..."); } for (int i = 0; i < r.length; i++) { final ContentProviderResult res = r[i]; final SyncStatus ss = syncStatuses.get(cpoPubUris.get(i)); if (ss == null) { Log.e(TAG, "can't get sync status for " + res.uri); continue; } syncMap.onPostSyncItem(mContext, account, ss.local, ss.remoteJson, res.count != null ? res.count == 1 : true); ss.state = SyncState.NOW_UP_TO_DATE; } if (DEBUG) { Log.d(TAG, "done running postSync handler."); } cpo.clear(); cpoPubUris.clear(); } if (Thread.interrupted()) { throw new InterruptedException(); } /* * Look through the SyncState.state values and find ones that need to be stored. */ for (final Map.Entry<String, SyncStatus> entry : syncStatuses.entrySet()) { if (Thread.interrupted()) { throw new InterruptedException(); } final String pubUri = entry.getKey(); final SyncStatus status = entry.getValue(); if (status.state == SyncState.REMOTE_ONLY) { if (DEBUG) { Log.d(TAG, pubUri + " is not yet stored locally, adding..."); } // update this so it's in the local timescale correctServerOffset(status.remoteCVs, JsonSyncableItem._CREATED_DATE, JsonSyncableItem._CREATED_DATE, localOffset); correctServerOffset(status.remoteCVs, JsonSyncableItem._SERVER_MODIFIED_DATE, JsonSyncableItem._MODIFIED_DATE, localOffset); final ContentProviderOperation.Builder b = ContentProviderOperation.newInsert(toSync); b.withValues(status.remoteCVs); cpo.add(b.build()); cpoPubUris.add(pubUri); syncResult.stats.numInserts++; } } /* * Execute the content provider operations in bulk. */ if (cpo.size() > 0) { if (DEBUG) { Log.d(TAG, "bulk inserting " + cpo.size() + " items..."); } final ContentProviderResult[] r = provider.applyBatch(cpo); if (DEBUG) { Log.d(TAG, "applyBatch completed. Processing results..."); } int successful = 0; for (int i = 0; i < r.length; i++) { final ContentProviderResult res = r[i]; if (res.uri == null) { syncResult.stats.numSkippedEntries++; Log.e(TAG, "result from content provider bulk operation returned null"); continue; } final String pubUri = cpoPubUris.get(i); final SyncStatus ss = syncStatuses.get(pubUri); if (ss == null) { syncResult.stats.numSkippedEntries++; Log.e(TAG, "could not find sync status for " + cpoPubUris.get(i)); continue; } ss.local = res.uri; if (DEBUG) { Log.d(TAG, "onPostSyncItem(" + res.uri + ", ...); pubUri: " + pubUri); } syncMap.onPostSyncItem(mContext, account, res.uri, ss.remoteJson, res.count != null ? res.count == 1 : true); ss.state = SyncState.NOW_UP_TO_DATE; successful++; } if (DEBUG) { Log.d(TAG, successful + " batch inserts successfully applied."); } } else { if (DEBUG) { Log.d(TAG, "no updates to perform."); } } /** * Look through all the items that we didn't already find on the server side, but which * still have a public uri. They should be checked to make sure they're not deleted. */ c = provider.query(toSync, SYNC_PROJECTION, ProviderUtils.addExtraWhere(selectionInverse, JsonSyncableItem._PUBLIC_URI + " NOT NULL"), selectionArgs, null); try { final int idCol = c.getColumnIndex(JsonSyncableItem._ID); final int pubUriCol = c.getColumnIndex(JsonSyncableItem._PUBLIC_URI); cpo.clear(); for (c.moveToFirst(); !c.isAfterLast(); c.moveToNext()) { final String pubUri = c.getString(pubUriCol); SyncStatus ss = syncStatuses.get(pubUri); final Uri item = isDir ? ContentUris.withAppendedId(toSyncWithoutQuerystring, c.getLong(idCol)) : toSync; if (ss == null) { ss = syncStatuses.get(item.toString()); } if (DEBUG) { Log.d(TAG, item + " was not found in the main list of items on the server (" + pubPath + "), but appears to be a child of " + toSync); if (ss != null) { Log.d(TAG, "found sync status for " + item + ": " + ss); } } if (ss != null) { switch (ss.state) { case ALREADY_UP_TO_DATE: case NOW_UP_TO_DATE: if (DEBUG) { Log.d(TAG, item + " is already up to date. No need to see if it was deleted."); } continue; case BOTH_UNKNOWN: if (DEBUG) { Log.d(TAG, item + " was found on both sides, but has an unknown sync status. Skipping..."); } continue; default: Log.w(TAG, "got an unexpected state for " + item + ": " + ss); } } else { ss = new SyncStatus(pubUri, SyncState.LOCAL_ONLY); ss.local = item; hr = mNetworkClient.head(pubUri); switch (hr.getStatusLine().getStatusCode()) { case 200: if (DEBUG) { Log.d(TAG, "HEAD " + pubUri + " returned 200"); } ss.state = SyncState.BOTH_UNKNOWN; break; case 404: if (DEBUG) { Log.d(TAG, "HEAD " + pubUri + " returned 404. Deleting locally..."); } ss.state = SyncState.DELETED_REMOTELY; final ContentProviderOperation deleteOp = ContentProviderOperation .newDelete(ContentUris.withAppendedId(toSyncWithoutQuerystring, c.getLong(idCol))) .build(); cpo.add(deleteOp); break; default: syncResult.stats.numIoExceptions++; Log.w(TAG, "HEAD " + pubUri + " got unhandled result: " + hr.getStatusLine()); } } syncStatuses.put(pubUri, ss); } // for cursor if (cpo.size() > 0) { final ContentProviderResult[] results = provider.applyBatch(cpo); for (final ContentProviderResult result : results) { if (result.count != 1) { throw new SyncException("Error deleting item"); } } } } finally { c.close(); } syncStatuses.clear(); mLastUpdated.markUpdated(toSync); return true; }
From source file:com.opengamma.examples.simulated.livedata.ExampleLiveDataServer.java
@Override protected void doDisconnect() { if (_executorService != null) { _executorService.shutdownNow();//from w ww. j a v a 2s . com try { _executorService.awaitTermination(1, TimeUnit.SECONDS); } catch (InterruptedException ex) { Thread.interrupted(); } } }
From source file:com.palantir.opensource.sysmon.linux.LinuxVMStatJMXWrapper.java
@Override public void run() { boolean wasInterrupted = Thread.interrupted(); try {/*from w w w . ja va 2 s. c o m*/ do { String line = null; try { if (vmstatStdout.ready()) { line = vmstatStdout.readLine(); } } catch (Exception e) { line = null; if (!shutdown) { log.warn("Caught exception while reading line.", e); } else { log.debug("Shutdown caused exception.", e); } } if (line != null) { try { LinuxVMStat dataBean = processLine(line); canonicalBean.takeValues(dataBean); // updates the JMX bean continue; } catch (LinuxMonitoringException e) { log.error(e); } } // throttle data reads. That way, if/when things are broken, we don't loop hard // on an error condition, log errors at full throttle try { if (!shutdown) { Thread.sleep(1000L); } } catch (InterruptedException e) { wasInterrupted = true; if (!shutdown) { log.warn("Interrupted while doing throttled sleep.", e); } else { log.debug("Shutdown caused exception.", e); } } } while (!shutdown); } finally { if (wasInterrupted) Thread.currentThread().interrupt(); cleanup(); } }
From source file:ddf.catalog.ftp.FtpServerStarter.java
private void waitForConnections() { FtpStatistics serverStatistics = ((DefaultFtpServer) server).getServerContext().getFtpStatistics(); int totalWait = 0; while (serverStatistics.getCurrentConnectionNumber() > 0) { LOGGER.debug("Waiting for {} connections to close before updating configuration", serverStatistics.getCurrentConnectionNumber()); try {/*from ww w . j a v a2s . c o m*/ if (totalWait <= maxSleepTimeMillis) { totalWait += resetWaitTimeMillis; Thread.sleep(resetWaitTimeMillis); } else { LOGGER.debug("Waited {} seconds for connections to close, updating FTP configuration", TimeUnit.MILLISECONDS.toSeconds(totalWait)); break; } } catch (InterruptedException e) { Thread.interrupted(); LOGGER.info("Thread interrupted while waiting for FTP connections to close", e); } } }
From source file:com.cloud.utils.script.Script.java
public String execute(OutputInterpreter interpreter) { String[] command = _command.toArray(new String[_command.size()]); if (_logger.isDebugEnabled()) { _logger.debug("Executing: " + buildCommandLine(command)); }// ww w .j ava2 s .c o m try { ProcessBuilder pb = new ProcessBuilder(command); pb.redirectErrorStream(true); if (_workDir != null) pb.directory(new File(_workDir)); _process = pb.start(); if (_process == null) { _logger.warn("Unable to execute: " + buildCommandLine(command)); return "Unable to execute the command: " + command[0]; } BufferedReader ir = new BufferedReader(new InputStreamReader(_process.getInputStream())); _thread = Thread.currentThread(); ScheduledFuture<String> future = null; if (_timeout > 0) { future = s_executors.schedule(this, _timeout, TimeUnit.MILLISECONDS); } Task task = null; if (interpreter != null && interpreter.drain()) { task = new Task(interpreter, ir); s_executors.execute(task); } while (true) { try { if (_process.waitFor() == 0) { _logger.debug("Execution is successful."); if (interpreter != null) { return interpreter.drain() ? task.getResult() : interpreter.interpret(ir); } else { // null return exitValue apparently return String.valueOf(_process.exitValue()); } } else { break; } } catch (InterruptedException e) { if (!_isTimeOut) { /* * This is not timeout, we are interrupted by others, * continue */ _logger.debug("We are interrupted but it's not a timeout, just continue"); continue; } TimedOutLogger log = new TimedOutLogger(_process); Task timedoutTask = new Task(log, ir); timedoutTask.run(); if (!_passwordCommand) { _logger.warn("Timed out: " + buildCommandLine(command) + ". Output is: " + timedoutTask.getResult()); } else { _logger.warn("Timed out: " + buildCommandLine(command)); } return ERR_TIMEOUT; } finally { if (future != null) { future.cancel(false); } Thread.interrupted(); } } _logger.debug("Exit value is " + _process.exitValue()); BufferedReader reader = new BufferedReader(new InputStreamReader(_process.getInputStream()), 128); String error; if (interpreter != null) { error = interpreter.processError(reader); } else { error = String.valueOf(_process.exitValue()); } if (_logger.isDebugEnabled()) { _logger.debug(error); } return error; } catch (SecurityException ex) { _logger.warn("Security Exception....not running as root?", ex); return stackTraceAsString(ex); } catch (Exception ex) { _logger.warn("Exception: " + buildCommandLine(command), ex); return stackTraceAsString(ex); } finally { if (_process != null) { IOUtils.closeQuietly(_process.getErrorStream()); IOUtils.closeQuietly(_process.getOutputStream()); IOUtils.closeQuietly(_process.getInputStream()); _process.destroy(); } } }
From source file:org.apache.openejb.activemq.JMS2AMQTest.java
@Test public void cdiListenerAPI() throws InterruptedException { final String text = TEXT + "4"; final AtomicReference<Throwable> error = new AtomicReference<>(); final CountDownLatch ready = new CountDownLatch(1); final CountDownLatch over = new CountDownLatch(1); new Thread() { {/* w w w . j av a 2 s. c o m*/ setName(JMS2AMQTest.class.getName() + ".cdiListenerAPI#receiver"); } @Override public void run() { final ContextsService contextsService = WebBeansContext.currentInstance().getContextsService(); contextsService.startContext(RequestScoped.class, null); try { final JMSConsumer consumer = context.createConsumer(destination3); consumer.setMessageListener(new MessageListener() { @Override public void onMessage(final Message message) { try { assertEquals(text, message.getBody(String.class)); } catch (final Throwable e) { error.set(e); } finally { over.countDown(); consumer.close(); } } }); ready.countDown(); } catch (final Throwable t) { error.set(t); } finally { try { over.await(1, TimeUnit.MINUTES); } catch (final InterruptedException e) { Thread.interrupted(); } contextsService.endContext(RequestScoped.class, null); } } }.start(); ready.await(1, TimeUnit.MINUTES); // now send the message try (final JMSContext context = cf.createContext()) { context.createProducer().send(destination3, text); } catch (final JMSRuntimeException ex) { fail(ex.getMessage()); } over.await(1, TimeUnit.MINUTES); // ensure we got the message and no exception final Throwable exception = error.get(); if (exception != null) { exception.printStackTrace(); } assertNull(exception == null ? "ok" : exception.getMessage(), exception); }
From source file:io.hops.ha.common.TransactionStateManager.java
private TransactionState getCurrentTransactionState(int rpcId, String callingFuncition, boolean priority) throws InterruptedException { while (true && !Thread.interrupted()) { int accepted = acceptedRPC.incrementAndGet(); if (priority || accepted < batchMaxSize) { lock.readLock().lock();// w w w . ja va 2 s. com try { transactionStateWrapper wrapper = new transactionStateWrapper( (TransactionStateImpl) currentTransactionState, TransactionState.TransactionType.RM, rpcId, callingFuncition); wrapper.incCounter(TransactionState.TransactionType.INIT); if (rpcId >= 0) { wrapper.addRPCId(rpcId); } curentRPCs.add(wrapper); return wrapper; } finally { lock.readLock().unlock(); } } else { acceptedRPC.decrementAndGet(); try { Thread.sleep(1); } catch (InterruptedException e) { LOG.warn(e, e); } } } throw new InterruptedException(); }
From source file:org.apache.hadoop.hbase.io.hfile.HFileReaderV2.java
/** * Opens a HFile. You must load the index before you can use it by calling * {@link #loadFileInfo()}.//from ww w . j ava 2 s. c o m * * @param path Path to HFile. * @param trailer File trailer. * @param fsdis input stream. * @param size Length of the stream. * @param cacheConf Cache configuration. * @param hfs * @param conf */ public HFileReaderV2(final Path path, final FixedFileTrailer trailer, final FSDataInputStreamWrapper fsdis, final long size, final CacheConfig cacheConf, final HFileSystem hfs, final Configuration conf) throws IOException { super(path, trailer, size, cacheConf, hfs, conf); this.conf = conf; trailer.expectMajorVersion(getMajorVersion()); validateMinorVersion(path, trailer.getMinorVersion()); this.hfileContext = createHFileContext(fsdis, fileSize, hfs, path, trailer); HFileBlock.FSReaderV2 fsBlockReaderV2 = new HFileBlock.FSReaderV2(fsdis, fileSize, hfs, path, hfileContext); this.fsBlockReader = fsBlockReaderV2; // upcast // Comparator class name is stored in the trailer in version 2. comparator = trailer.createComparator(); dataBlockIndexReader = new HFileBlockIndex.BlockIndexReader(comparator, trailer.getNumDataIndexLevels(), this); metaBlockIndexReader = new HFileBlockIndex.BlockIndexReader(KeyValue.RAW_COMPARATOR, 1); // Parse load-on-open data. HFileBlock.BlockIterator blockIter = fsBlockReaderV2.blockRange(trailer.getLoadOnOpenDataOffset(), fileSize - trailer.getTrailerSize()); // Data index. We also read statistics about the block index written after // the root level. dataBlockIndexReader.readMultiLevelIndexRoot(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), trailer.getDataIndexCount()); // Meta index. metaBlockIndexReader.readRootIndex(blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX), trailer.getMetaIndexCount()); // File info fileInfo = new FileInfo(); fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream()); lastKey = fileInfo.get(FileInfo.LASTKEY); avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN)); avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN)); byte[] keyValueFormatVersion = fileInfo.get(HFileWriterV2.KEY_VALUE_VERSION); includesMemstoreTS = keyValueFormatVersion != null && Bytes.toInt(keyValueFormatVersion) == HFileWriterV2.KEY_VALUE_VER_WITH_MEMSTORE; fsBlockReaderV2.setIncludesMemstoreTS(includesMemstoreTS); if (includesMemstoreTS) { decodeMemstoreTS = Bytes.toLong(fileInfo.get(HFileWriterV2.MAX_MEMSTORE_TS_KEY)) > 0; } // Read data block encoding algorithm name from file info. dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo); fsBlockReaderV2.setDataBlockEncoder(dataBlockEncoder); // Store all other load-on-open blocks for further consumption. HFileBlock b; while ((b = blockIter.nextBlock()) != null) { loadOnOpenBlocks.add(b); } // Prefetch file blocks upon open if requested if (cacheConf.shouldPrefetchOnOpen()) { PrefetchExecutor.request(path, new Runnable() { public void run() { try { long offset = 0; long end = fileSize - getTrailer().getTrailerSize(); HFileBlock prevBlock = null; while (offset < end) { if (Thread.interrupted()) { break; } long onDiskSize = -1; if (prevBlock != null) { onDiskSize = prevBlock.getNextBlockOnDiskSizeWithHeader(); } HFileBlock block = readBlock(offset, onDiskSize, true, false, false, false, null, null); prevBlock = block; offset += block.getOnDiskSizeWithHeader(); } } catch (IOException e) { // IOExceptions are probably due to region closes (relocation, etc.) if (LOG.isTraceEnabled()) { LOG.trace("Exception encountered while prefetching " + path + ":", e); } } catch (Exception e) { // Other exceptions are interesting LOG.warn("Exception encountered while prefetching " + path + ":", e); } finally { PrefetchExecutor.complete(path); } } }); } }
From source file:net.iponweb.hadoop.streaming.parquet.TextRecordWriterWrapper.java
@Override public void write(Text key, Text value) throws IOException { Group grp = factory.newGroup(); String[] strK = key.toString().split(TAB, -1); String[] strV = value == null ? new String[0] : value.toString().split(TAB, -1); String kv_combined[] = (String[]) ArrayUtils.addAll(strK, strV); Iterator<PathAction> ai = recorder.iterator(); Stack<Group> groupStack = new Stack<>(); groupStack.push(grp);/*from w ww . ja v a 2s . com*/ int i = 0; try { while (ai.hasNext()) { PathAction a = ai.next(); switch (a.getAction()) { case GROUPEND: grp = groupStack.pop(); break; case GROUPSTART: groupStack.push(grp); grp = grp.addGroup(a.getName()); break; case FIELD: String s = null; PrimitiveType.PrimitiveTypeName primType = a.getType(); String colName = a.getName(); if (i < kv_combined.length) s = kv_combined[i++]; if (s == null) { if (a.getRepetition() == Type.Repetition.OPTIONAL) { i++; continue; } s = primType == PrimitiveType.PrimitiveTypeName.BINARY ? "" : "0"; } // If we have 'repeated' field, assume that we should expect JSON-encoded array // Convert array and append all values int repetition = 1; boolean repeated = false; ArrayList<String> s_vals = null; if (a.getRepetition() == Type.Repetition.REPEATED) { repeated = true; s_vals = new ArrayList<>(); ObjectMapper mapper = new ObjectMapper(); JsonNode node = mapper.readTree(s); Iterator<JsonNode> itr = node.iterator(); repetition = 0; while (itr.hasNext()) { String o; switch (primType) { case BINARY: o = itr.next().getTextValue(); // No array-of-objects! break; case BOOLEAN: o = String.valueOf(itr.next().getBooleanValue()); break; default: o = String.valueOf(itr.next().getNumberValue()); } s_vals.add(o); repetition++; } } for (int j = 0; j < repetition; j++) { if (repeated) // extract new s s = s_vals.get(j); try { switch (primType) { case INT32: grp.append(colName, new Double(Double.parseDouble(s)).intValue()); break; case INT64: case INT96: grp.append(colName, new Double(Double.parseDouble(s)).longValue()); break; case DOUBLE: grp.append(colName, Double.parseDouble(s)); break; case FLOAT: grp.append(colName, Float.parseFloat(s)); break; case BOOLEAN: grp.append(colName, s.equalsIgnoreCase("true") || s.equals("1")); break; case BINARY: grp.append(colName, Binary.fromString(s)); break; default: throw new RuntimeException("Can't handle type " + primType); } } catch (NumberFormatException e) { grp.append(colName, 0); } } } } realWriter.write(null, (SimpleGroup) grp); } catch (InterruptedException e) { Thread.interrupted(); throw new IOException(e); } catch (Exception e) { ByteArrayOutputStream out = new ByteArrayOutputStream(); e.printStackTrace(new PrintStream(out)); throw new RuntimeException("Failed on record " + grp + ", schema=" + schema + ", path action=" + recorder + " exception = " + e.getClass() + ", msg=" + e.getMessage() + ", cause=" + e.getCause() + ", trace=" + out.toString()); } }
From source file:at.alladin.rmbt.android.test.RMBTTask.java
private void doInBackground() { try {/*from w w w . j ava 2 s .c o m*/ boolean error = false; connectionError.set(false); TestResult result = null; QoSResultCollector qosResult = null; try { final String uuid = fullInfo.getUUID(); final String controlServer = ConfigHelper.getControlServerName(context); final int controlPort = ConfigHelper.getControlServerPort(context); final boolean controlSSL = ConfigHelper.isControlSeverSSL(context); final ArrayList<String> geoInfo = fullInfo.getCurLocation(); client = RMBTClient.getInstance(controlServer, null, controlPort, controlSSL, geoInfo, uuid, Config.RMBT_CLIENT_TYPE, Config.RMBT_CLIENT_NAME, fullInfo.getInfo("CLIENT_SOFTWARE_VERSION"), null, fullInfo.getInitialInfo()); if (client != null) { /* * Example on how to implement the information collector tool: * * */ final InformationCollectorTool infoTool = new InformationCollectorTool(TimeUnit.NANOSECONDS, TimeUnit.NANOSECONDS.convert(120, TimeUnit.SECONDS)); infoTool.addCollector(new CpuStatCollector(new CpuStatAndroidImpl(), TimeUnit.NANOSECONDS.convert(1, TimeUnit.SECONDS))); infoTool.addCollector(new MemInfoCollector(new MemInfoAndroidImpl(), TimeUnit.NANOSECONDS.convert(1, TimeUnit.SECONDS))); client.setInformationCollectorTool(infoTool); client.startInformationCollectorTool(); ///////////////////////////////////////////////////////////////// client.setTrafficService(new TrafficServiceImpl()); final ControlServerConnection controlConnection = client.getControlConnection(); if (controlConnection != null) { fullInfo.setUUID(controlConnection.getClientUUID()); fullInfo.setTestServerName(controlConnection.getServerName()); } } } catch (final Exception e) { e.printStackTrace(); error = true; } if (error || client == null) { connectionError.set(true); } else { if (client.getStatus() != TestStatus.ERROR) { try { if (Thread.interrupted() || cancelled.get()) throw new InterruptedException(); Log.d(LOG_TAG, "runTest RMBTTask=" + this); result = client.runTest(); final ControlServerConnection controlConnection = client.getControlConnection(); final InformationCollectorTool infoCollectorTool = client.getInformationCollectorTool(); if (result != null && !fullInfo.getIllegalNetworkTypeChangeDetcted()) { final JSONObject infoObject = fullInfo .getResultValues(controlConnection.getStartTimeNs()); if (infoCollectorTool != null) { infoCollectorTool.stop(); infoObject.put("extended_test_stat", infoCollectorTool.getJsonObject(true, client.getControlConnection().getStartTimeNs())); } infoObject.put("publish_public_data", ConfigHelper.isInformationCommissioner(context)); client.sendResult(infoObject); } else { error = true; } } catch (final Exception e) { e.printStackTrace(); } finally { client.shutdown(); } } else { System.err.println(client.getErrorMsg()); error = true; } //client.shutdown(); setPreviousTestStatus(); QualityOfServiceTest qosTest = null; boolean runQoS = (client.getTaskDescList() != null && client.getTaskDescList().size() >= 1); //run qos test: if (runQoS && !error && !cancelled.get()) { try { TestSettings qosTestSettings = new TestSettings(); qosTestSettings.setCacheFolder(context.getCacheDir()); qosTestSettings.setWebsiteTestService(new WebsiteTestServiceImpl(context)); qosTestSettings.setTrafficService(new TrafficServiceImpl()); qosTestSettings.setTracerouteServiceClazz(TracerouteAndroidImpl.class); qosTestSettings.setStartTimeNs(getRmbtClient().getControlConnection().getStartTimeNs()); qosTestSettings.setUseSsl(ConfigHelper.isQoSSeverSSL(context)); qosTest = new QualityOfServiceTest(client, qosTestSettings); qosReference.set(qosTest); client.setStatus(TestStatus.QOS_TEST_RUNNING); qosResult = qosTest.call(); InformationCollector.qoSResult = qosResult; if (!cancelled.get()) { if (qosResult != null && !qosTest.getStatus().equals(QoSTestEnum.ERROR)) { client.sendQoSResult(qosResult); } } } catch (Exception e) { e.printStackTrace(); error = true; } } if (qosTest != null && !cancelled.get() && qosTest.getStatus().equals(QoSTestEnum.QOS_FINISHED)) { if (ConfigHelper.isNDT(context)) { qosTest.setStatus(QoSTestEnum.NDT_RUNNING); runNDT(); } qosTest.setStatus(QoSTestEnum.STOP); } } } catch (final Exception e) { client.setStatus(TestStatus.ERROR); e.printStackTrace(); Thread.currentThread().interrupt(); } finally { try { if (client != null) { client.stopInformationCollectorTool(); final TestStatus status = client.getStatus(); if (!(status == TestStatus.ABORTED || status == TestStatus.ERROR)) client.setStatus(TestStatus.END); } } catch (Exception e) { } } }