List of usage examples for com.google.common.collect Iterables partition
public static <T> Iterable<List<T>> partition(final Iterable<T> iterable, final int size)
From source file:com.palantir.atlasdb.transaction.impl.SnapshotTransaction.java
@Override public Iterable<BatchingVisitable<RowResult<byte[]>>> getRanges(final String tableName, Iterable<RangeRequest> rangeRequests) { checkGetPreconditions(tableName);//from ww w .j a v a2 s . c om if (perfLogger.isDebugEnabled()) { perfLogger.debug("Passed {} ranges to getRanges({}, {})", Iterables.size(rangeRequests), tableName, rangeRequests); } return FluentIterable.from(Iterables.partition(rangeRequests, BATCH_SIZE_GET_FIRST_PAGE)) .transformAndConcat(new Function<List<RangeRequest>, List<BatchingVisitable<RowResult<byte[]>>>>() { @Override public List<BatchingVisitable<RowResult<byte[]>>> apply(List<RangeRequest> input) { Stopwatch timer = Stopwatch.createStarted(); Map<RangeRequest, TokenBackedBasicResultsPage<RowResult<Value>, byte[]>> firstPages = keyValueService .getFirstBatchForRanges(tableName, input, getStartTimestamp()); validateExternalAndCommitLocksIfNecessary(tableName); final SortedMap<Cell, byte[]> postFiltered = postFilterPages(tableName, firstPages.values()); List<BatchingVisitable<RowResult<byte[]>>> ret = Lists .newArrayListWithCapacity(input.size()); for (final RangeRequest rangeRequest : input) { TokenBackedBasicResultsPage<RowResult<Value>, byte[]> prePostFilter = firstPages .get(rangeRequest); final byte[] nextStartRowName = getNextStartRowName(rangeRequest, prePostFilter); final List<Entry<Cell, byte[]>> mergeIterators = getPostfilteredWithLocalWrites( tableName, postFiltered, rangeRequest, prePostFilter.getResults(), nextStartRowName); ret.add(new AbstractBatchingVisitable<RowResult<byte[]>>() { @Override protected <K extends Exception> void batchAcceptSizeHint(int batchSizeHint, ConsistentVisitor<RowResult<byte[]>, K> v) throws K { checkGetPreconditions(tableName); final Iterator<RowResult<byte[]>> rowResults = Cells .createRowView(mergeIterators); while (rowResults.hasNext()) { if (!v.visit(ImmutableList.of(rowResults.next()))) { return; } } if (nextStartRowName.length == 0) { return; } RangeRequest newRange = rangeRequest.getBuilder() .startRowInclusive(nextStartRowName).build(); getRange(tableName, newRange).batchAccept(batchSizeHint, v); } }); } log.info("Processed {} range requests for {} in {}ms", input.size(), tableName, timer.elapsed(TimeUnit.MILLISECONDS)); return ret; } }); }
From source file:com.blacklocus.metrics.CloudWatchReporter.java
public void realReport(SortedMap<String, Gauge> gauges, SortedMap<String, Counter> counters, SortedMap<String, Histogram> histograms, SortedMap<String, Meter> meters, SortedMap<String, Timer> timers) { try {//from ww w .j av a 2 s . co m // Just an estimate to reduce resizing. List<MetricDatum> data = new ArrayList<MetricDatum>( gauges.size() + counters.size() + meters.size() + 2 * histograms.size() + 2 * timers.size()); // Translate various metric classes to MetricDatum for (Map.Entry<String, Gauge> gaugeEntry : gauges.entrySet()) { reportGauge(gaugeEntry, typeDimValGauge, data); } for (Map.Entry<String, Counter> counterEntry : counters.entrySet()) { reportCounter(counterEntry, typeDimValCounterCount, data); } for (Map.Entry<String, Meter> meterEntry : meters.entrySet()) { reportCounter(meterEntry, typeDimValMeterCount, data); } for (Map.Entry<String, Histogram> histogramEntry : histograms.entrySet()) { reportCounter(histogramEntry, typeDimValHistoSamples, data); reportSampling(histogramEntry, typeDimValHistoStats, 1.0, data); } for (Map.Entry<String, Timer> timerEntry : timers.entrySet()) { reportCounter(timerEntry, typeDimValTimerSamples, data); reportSampling(timerEntry, typeDimValTimerStats, 0.000001, data); // nanos -> millis } // Filter out unreportable entries. Collection<MetricDatum> nonEmptyData = Collections2.filter(data, new Predicate<MetricDatum>() { @Override public boolean apply(MetricDatum input) { if (input == null) { return false; } else if (input.getStatisticValues() != null) { // CloudWatch rejects any Statistic Sets with sample count == 0, which it probably should reject. return input.getStatisticValues().getSampleCount() > 0; } return true; } }); // Whether to use local "now" (true, new Date()) or cloudwatch service "now" (false, leave null). if (timestampLocal) { Date now = new Date(); for (MetricDatum datum : nonEmptyData) { datum.withTimestamp(now); } } // Finally, apply any user-level filter. Collection<MetricDatum> filtered = Collections2.filter(nonEmptyData, reporterFilter); // Each CloudWatch API request may contain at maximum 20 datums. Break into partitions of 20. Iterable<List<MetricDatum>> dataPartitions = Iterables.partition(filtered, 20); List<Future<?>> cloudWatchFutures = Lists.newArrayListWithExpectedSize(filtered.size()); // Submit asynchronously with threads. for (List<MetricDatum> dataSubset : dataPartitions) { cloudWatchFutures.add(cloudWatch.putMetricDataAsync( new PutMetricDataRequest().withNamespace(metricNamespace).withMetricData(dataSubset))); } // Wait for CloudWatch putMetricData futures to be fulfilled. for (Future<?> cloudWatchFuture : cloudWatchFutures) { // We can't let an exception leak out of here, or else the reporter will cease running as described in // java.util.concurrent.ScheduledExecutorService.scheduleAtFixedRate(Runnable, long, long, TimeUnit unit) try { // See what happened in case of an error. cloudWatchFuture.get(); } catch (Exception e) { LOG.error("Exception reporting metrics to CloudWatch. The data in this CloudWatch API request " + "may have been discarded, did not make it to CloudWatch.", e); } } LOG.debug("Sent {} metric data to CloudWatch. namespace: {}", filtered.size(), metricNamespace); } catch (RuntimeException e) { LOG.error("Error marshalling CloudWatch metrics.", e); } }
From source file:ei.ne.ke.cassandra.cql3.AstyanaxCql3Repository.java
/** * {@inheritDoc}/* w w w .j a v a2s. co m*/ */ @Override public synchronized void delete(Iterable<? extends T> entities) { int count = Iterables.size(entities); List<Callable<List<? extends T>>> todo = Lists.newArrayListWithExpectedSize(count / batchSize); for (Iterable<? extends T> partition : Iterables.partition(entities, batchSize)) { todo.add(new Deleter(partition)); } try { List<Future<List<? extends T>>> futureResults = executorService.invokeAll(todo); waitUntilCompletion(futureResults); } catch (InterruptedException e) { Thread.interrupted(); } }
From source file:com.palantir.atlasdb.keyvalue.cassandra.CQLKeyValueService.java
private Map<Cell, Long> getLatestTimestampsInternal(final String tableName, Map<Cell, Long> timestampByCell) throws Exception { final CassandraKeyValueServiceConfig config = configManager.getConfig(); int fetchBatchCount = config.fetchBatchCount(); Iterable<List<Cell>> partitions = Iterables.partition(timestampByCell.keySet(), fetchBatchCount); int numPartitions = (timestampByCell.size() / fetchBatchCount) + (timestampByCell.size() % fetchBatchCount > 0 ? 1 : 0); List<Future<Map<Cell, Long>>> futures = Lists.newArrayListWithCapacity(numPartitions); final String loadOnlyTsQuery = "SELECT " + CassandraConstants.ROW_NAME + ", " + CassandraConstants.COL_NAME_COL + ", " + CassandraConstants.TS_COL + " FROM " + getFullTableName(tableName) + " " + "WHERE " + CassandraConstants.ROW_NAME + " = ? AND " + CassandraConstants.COL_NAME_COL + " = ? LIMIT 1"; if (timestampByCell.size() > fetchBatchCount) { log.warn("Re-batching in getLatestTimestamps a call to " + tableName + " that attempted to multiget " + timestampByCell.size() + " cells; this may indicate overly-large batching on a higher level.\n" + CassandraKeyValueServices.getFilteredStackTrace("com.palantir")); }//from w ww .java 2 s. c om for (final List<Cell> partition : partitions) { futures.add(executor.submit(new Callable<Map<Cell, Long>>() { @Override public Map<Cell, Long> call() throws Exception { PreparedStatement preparedStatement = getPreparedStatement(tableName, loadOnlyTsQuery, session); preparedStatement.setConsistencyLevel(readConsistency); List<ResultSetFuture> resultSetFutures = Lists.newArrayListWithExpectedSize(partition.size()); for (Cell c : partition) { BoundStatement boundStatement = preparedStatement.bind(); boundStatement.setBytes(CassandraConstants.ROW_NAME, ByteBuffer.wrap(c.getRowName())); boundStatement.setBytes(CassandraConstants.COL_NAME_COL, ByteBuffer.wrap(c.getColumnName())); resultSetFutures.add(session.executeAsync(boundStatement)); } Map<Cell, Long> res = Maps.newHashMapWithExpectedSize(partition.size()); for (ResultSetFuture resultSetFuture : resultSetFutures) { ResultSet resultSet = resultSetFuture.getUninterruptibly(); for (Row row : resultSet.all()) { res.put(Cell.create(CQLKeyValueServices.getRowName(row), CQLKeyValueServices.getColName(row)), CQLKeyValueServices.getTs(row)); } CQLKeyValueServices.logTracedQuery(loadOnlyTsQuery, resultSet, session, cqlStatementCache.NORMAL_QUERY); } return res; } })); } Map<Cell, Long> res = Maps.newHashMapWithExpectedSize(timestampByCell.size()); for (Future<Map<Cell, Long>> f : futures) { try { res.putAll(f.get()); } catch (InterruptedException e) { throw Throwables.throwUncheckedException(e); } catch (ExecutionException e) { Throwables.throwIfInstance(e, Error.class); throw Throwables.throwUncheckedException(e.getCause()); } } return res; }
From source file:org.candlepin.model.EntitlementCurator.java
/** * Marks the given entitlements as dirty; forcing a regeneration the next time it is requested. * * @param entitlementIds/* w w w . jav a 2 s .com*/ * A collection of IDs of the entitlements to mark dirty * * @return * The number of certificates updated */ @Transactional public int markEntitlementsDirty(Iterable<String> entitlementIds) { int count = 0; if (entitlementIds != null && entitlementIds.iterator().hasNext()) { Iterable<List<String>> blocks = Iterables.partition(entitlementIds, AbstractHibernateCurator.IN_OPERATOR_BLOCK_SIZE); String hql = "UPDATE Entitlement SET dirty = true WHERE id IN (:entIds)"; Query query = this.getEntityManager().createQuery(hql); for (List<String> block : blocks) { count += query.setParameter("entIds", block).executeUpdate(); } } return count; }
From source file:com.b2international.snowowl.snomed.datastore.id.cis.CisSnomedIdentifierService.java
private Map<String, SctId> readSctIds(final Set<String> componentIds) { HttpPost bulkRequest = null;//from w ww. ja va 2 s . c om HttpGet singleRequest = null; try { if (componentIds.size() > 1) { LOGGER.debug("Sending bulk component ID get request."); final ImmutableMap.Builder<String, SctId> resultBuilder = ImmutableMap.builder(); for (final Collection<String> ids : Iterables.partition(componentIds, BULK_LIMIT)) { final String idsAsString = Joiner.on(',').join(ids); final ObjectNode idsAsJson = mapper.createObjectNode().put("sctids", idsAsString); bulkRequest = client.httpPost(String.format("sct/bulk/ids/?token=%s", getToken()), idsAsJson); final String response = execute(bulkRequest); final SctId[] sctIds = mapper.readValue(response, SctId[].class); final Map<String, SctId> sctIdMap = Maps.uniqueIndex(Arrays.asList(sctIds), SctId::getSctid); resultBuilder.putAll(sctIdMap); } return resultBuilder.build(); } else { final String componentId = Iterables.getOnlyElement(componentIds); LOGGER.debug(String.format("Sending component ID %s get request.", componentId)); singleRequest = httpGet(String.format("sct/ids/%s?token=%s", componentId, getToken())); final String response = execute(singleRequest); final SctId sctId = mapper.readValue(response, SctId.class); return ImmutableMap.of(sctId.getSctid(), sctId); } } catch (IOException e) { throw new SnowowlRuntimeException("Exception while getting all IDs", e); } finally { release(bulkRequest); release(singleRequest); } }
From source file:org.candlepin.model.AbstractHibernateCurator.java
/** * While hibernate does not have limits over how many values can be used in an in clause, * the underlying databases sometimes do. This method builds an unbounded in clause * by building logical or expressions out of batches of in-clauses. * * @param expression the string expression against which we are searching values * @param values the values being searched for the expression * @return the unbounded in criterion as described above *///from ww w . j av a 2s. c o m public <T extends Object> Criterion unboundedInCriterion(String expression, Iterable<T> values) { Criterion criterion = null; if (values == null || !values.iterator().hasNext()) { throw new IllegalArgumentException("values is null or empty"); } for (List<T> block : Iterables.partition(values, IN_OPERATOR_BLOCK_SIZE)) { criterion = (criterion == null) ? Restrictions.in(expression, block) : Restrictions.or(criterion, Restrictions.in(expression, block)); } return criterion; }
From source file:edu.umn.msi.tropix.persistence.dao.hibernate.TropixObjectDaoImpl.java
public Set<String> getFilesObjectIds(final Set<String> fileIds) { final Set<String> objectIds = Sets.newHashSet(); for (final Iterable<String> fileIdsPartition : Iterables.partition(fileIds, 100)) { final Query query = super.getSession() .createQuery("select f.id from TropixFile f where f.fileId in (:fileIds)"); query.setParameterList("fileIds", Lists.newArrayList(fileIdsPartition)); @SuppressWarnings("unchecked") List<String> partitionObjectIds = query.list(); objectIds.addAll(partitionObjectIds); }//from www .j a v a 2 s. c o m return objectIds; }
From source file:com.palantir.atlasdb.cleaner.Scrubber.java
/** * @return number of cells read from _scrub table *//*from w w w .j a v a 2 s . c o m*/ private int scrubSomeCells(SortedMap<Long, Multimap<String, Cell>> scrubTimestampToTableNameToCell, final TransactionManager txManager, long maxScrubTimestamp) { // Don't call expensive toString() if trace logging is off if (log.isTraceEnabled()) { log.trace("Attempting to scrub cells: " + scrubTimestampToTableNameToCell); } if (log.isInfoEnabled()) { int numCells = 0; Set<String> tables = Sets.newHashSet(); for (Multimap<String, Cell> v : scrubTimestampToTableNameToCell.values()) { tables.addAll(v.keySet()); numCells += v.size(); } log.info("Attempting to scrub " + numCells + " cells from tables " + tables); } if (scrubTimestampToTableNameToCell.size() == 0) { return 0; // No cells left to scrub } Multimap<Long, Cell> toRemoveFromScrubQueue = HashMultimap.create(); int numCellsReadFromScrubTable = 0; List<Future<Void>> scrubFutures = Lists.newArrayList(); for (Map.Entry<Long, Multimap<String, Cell>> entry : scrubTimestampToTableNameToCell.entrySet()) { final long scrubTimestamp = entry.getKey(); final Multimap<String, Cell> tableNameToCell = entry.getValue(); numCellsReadFromScrubTable += tableNameToCell.size(); long commitTimestamp = getCommitTimestampRollBackIfNecessary(scrubTimestamp, tableNameToCell); if (commitTimestamp >= maxScrubTimestamp) { // We cannot scrub this yet because not all transactions can read this value. continue; } else if (commitTimestamp != TransactionConstants.FAILED_COMMIT_TS) { // This is CRITICAL; don't scrub if the hard delete transaction didn't actually finish // (we still remove it from the _scrub table with the call to markCellsAsScrubbed though), // or else we could cause permanent data loss if the hard delete transaction failed after // queuing cells to scrub but before successfully committing for (final List<Entry<String, Cell>> batch : Iterables.partition(tableNameToCell.entries(), batchSizeSupplier.get())) { final Multimap<String, Cell> batchMultimap = HashMultimap.create(); for (Entry<String, Cell> e : batch) { batchMultimap.put(e.getKey(), e.getValue()); } scrubFutures.add(exec.submit(new Callable<Void>() { @Override public Void call() throws Exception { scrubCells(txManager, batchMultimap, scrubTimestamp, aggressiveScrub ? TransactionType.AGGRESSIVE_HARD_DELETE : TransactionType.HARD_DELETE); return null; } })); } } toRemoveFromScrubQueue.putAll(scrubTimestamp, tableNameToCell.values()); } for (Future<Void> future : scrubFutures) { Futures.getUnchecked(future); } Multimap<Cell, Long> cellToScrubTimestamp = HashMultimap.create(); scrubberStore.markCellsAsScrubbed(Multimaps.invertFrom(toRemoveFromScrubQueue, cellToScrubTimestamp), batchSizeSupplier.get()); if (log.isTraceEnabled()) { log.trace("Finished scrubbing cells: " + scrubTimestampToTableNameToCell); } if (log.isInfoEnabled()) { Set<String> tables = Sets.newHashSet(); for (Multimap<String, Cell> v : scrubTimestampToTableNameToCell.values()) { tables.addAll(v.keySet()); } long minTimestamp = Collections.min(scrubTimestampToTableNameToCell.keySet()); long maxTimestamp = Collections.max(scrubTimestampToTableNameToCell.keySet()); log.info("Finished scrubbing " + numCellsReadFromScrubTable + " cells at " + scrubTimestampToTableNameToCell.size() + " timestamps (" + minTimestamp + "..." + maxTimestamp + ") from tables " + tables); } return numCellsReadFromScrubTable; }
From source file:org.candlepin.model.AbstractHibernateCurator.java
public List<E> lockAndLoadBatch(Iterable<String> ids, String entityName, String keyName) { List<E> result = new LinkedList<E>(); if (ids != null && ids.iterator().hasNext()) { StringBuilder hql = new StringBuilder("SELECT obj FROM ").append(entityName).append(" obj WHERE ") .append(keyName).append(" IN (:ids)"); Query query = this.getEntityManager().createQuery(hql.toString()) .setLockMode(LockModeType.PESSIMISTIC_WRITE); for (List<String> block : Iterables.partition(ids, IN_OPERATOR_BLOCK_SIZE)) { query.setParameter("ids", block); result.addAll((List<E>) query.getResultList()); }//from w w w . j a va2s .c om //In some situations, even after locking the entity we //got stale in the entity e.g. Pool.consumed //This refresh reloads the entity after the lock has //been issued. for (E e : result) { getEntityManager().refresh(e); } } return result; }