Example usage for com.google.common.collect Iterables partition

Introduction

In this page you can find the example usage for com.google.common.collect Iterables partition.

Prototype

public static <T> Iterable<List<T>> partition(final Iterable<T> iterable, final int size) 

Source Link

Document

Divides an iterable into unmodifiable sublists of the given size (the final iterable may be smaller).

Usage

From source file:kr.debop4j.core.parallelism.Parallels.java

/**
 *  ? ,  ? ./*from  w  ww  .j  a v  a  2  s  .co  m*/
 *
 * @param elements  ??
 * @param action    
 */
public static <T> void runPartitions(final Iterable<T> elements, final Action1<List<T>> action) {
    shouldNotBeNull(elements, "elements");
    shouldNotBeNull(action, "function");
    if (isDebugEnabled)
        log.debug(" ? ... workerCount=[{}]", getWorkerCount());

    ExecutorService executor = Executors.newFixedThreadPool(getWorkerCount());

    try {
        List<T> elemList = Lists.newArrayList(elements);
        int partitionSize = getPartitionSize(elemList.size(), getWorkerCount());
        Iterable<List<T>> partitions = Iterables.partition(elemList, partitionSize);
        List<Callable<Void>> tasks = Lists.newLinkedList();

        for (final List<T> partition : partitions) {
            Callable<Void> task = new Callable<Void>() {
                @Override
                public Void call() throws Exception {
                    action.perform(partition);
                    return null;
                }
            };
            tasks.add(task);
        }
        //  
        List<Future<Void>> results = executor.invokeAll(tasks);

        for (Future<Void> result : results)
            result.get();

        if (isDebugEnabled)
            log.debug(" ?  . workCount=[{}]");
    } catch (Exception e) {
        log.error("???    ?.", e);
        throw new RuntimeException(e);
    } finally {
        executor.shutdown();
    }
}

---

From source file:org.apache.beam.sdk.io.aws.s3.S3FileSystem.java

@Override
protected void delete(Collection<S3ResourceId> resourceIds) throws IOException {
    List<S3ResourceId> nonDirectoryPaths = resourceIds.stream()
            .filter(s3ResourceId -> !s3ResourceId.isDirectory()).collect(Collectors.toList());
    Multimap<String, String> keysByBucket = ArrayListMultimap.create();
    for (S3ResourceId path : nonDirectoryPaths) {
        keysByBucket.put(path.getBucket(), path.getKey());
    }//from w  w  w  . jav  a2s  .  c o  m

    List<Callable<Void>> tasks = new ArrayList<>();
    for (final String bucket : keysByBucket.keySet()) {
        for (final List<String> keysPartition : Iterables.partition(keysByBucket.get(bucket),
                MAX_DELETE_OBJECTS_PER_REQUEST)) {
            tasks.add(() -> {
                delete(bucket, keysPartition);
                return null;
            });
        }
    }

    callTasks(tasks);
}

---

From source file:ei.ne.ke.cassandra.cql3.AstyanaxCql3Repository.java

/**
 * {@inheritDoc}//from  w w  w.j  a  va 2s.c  o  m
 */
@Override
public List<T> findAll(Iterable<ID> ids) {
    List<T> entities = Lists.newArrayListWithExpectedSize(Iterables.size(ids));
    List<Callable<List<T>>> todo = Lists.newArrayListWithExpectedSize(Iterables.size(entities) / batchSize);
    Iterable<List<ID>> partitions = Iterables.partition(ids, batchSize);
    for (List<ID> partition : partitions) {
        todo.add(new Finder(partition));
    }
    try {
        List<Future<List<T>>> futureResults = executorService.invokeAll(todo);
        for (Future<List<T>> futureResult : futureResults) {
            try {
                entities.addAll(futureResult.get());
            } catch (ExecutionException e) {
                throw new RuntimeException(e);
            }
        }
    } catch (InterruptedException e) {
        Thread.interrupted();
    }
    return entities;
}

---

From source file:com.palantir.atlasdb.cleaner.Scrubber.java

private void scrubCells(TransactionManager txManager, Multimap<String, Cell> tableNameToCells,
        long scrubTimestamp, Transaction.TransactionType transactionType) {
    for (Entry<String, Collection<Cell>> entry : tableNameToCells.asMap().entrySet()) {
        String tableName = entry.getKey();
        if (log.isInfoEnabled()) {
            log.info("Attempting to immediately scrub " + entry.getValue().size() + " cells from table "
                    + tableName);/*from   w ww  . ja  v a  2s  . co  m*/
        }
        for (List<Cell> cells : Iterables.partition(entry.getValue(), batchSizeSupplier.get())) {
            Multimap<Cell, Long> timestampsToDelete = HashMultimap.create(
                    keyValueService.getAllTimestamps(tableName, ImmutableSet.copyOf(cells), scrubTimestamp));
            for (Cell cell : ImmutableList.copyOf(timestampsToDelete.keySet())) {
                // Don't scrub garbage collection sentinels
                timestampsToDelete.remove(cell, Value.INVALID_VALUE_TIMESTAMP);
            }
            // If transactionType == TransactionType.AGGRESSIVE_HARD_DELETE this might
            // force other transactions to abort or retry
            deleteCellsAtTimestamps(txManager, tableName, timestampsToDelete, transactionType);
        }
        if (log.isInfoEnabled()) {
            log.info("Immediately scrubbed " + entry.getValue().size() + " cells from table " + tableName);
        }
    }
}

---

From source file:jp.classmethod.aws.dynamodb.DynamoDbRepository.java

private List<E> findAll(Iterable<AttributeValue> ids, boolean useParallelBatches) {
    Preconditions.checkNotNull(ids, "ids may not be null");
    List<AttributeValue> idList = Lists.newArrayList(ids);
    if (idList.isEmpty()) {
        return new ArrayList<>();
    }// w  w w  . ja v  a2  s. com
    List<Map<String, AttributeValue>> resultantItems = new ArrayList<>();

    StreamSupport.stream(Iterables.partition(idList, 25).spliterator(), useParallelBatches).forEach(inner -> {
        BatchGetItemRequest req = new BatchGetItemRequest();
        KeysAndAttributes keysAndAttributes = new KeysAndAttributes();
        keysAndAttributes.setConsistentRead(true);
        keysAndAttributes.setKeys(
                inner.stream().map(id -> ImmutableMap.of(hashKeyName, id)).collect(Collectors.toList()));
        String tableName = tableName();
        req.withRequestItems(ImmutableMap.of(tableName, keysAndAttributes));

        BatchGetItemResult result;

        do {
            try {
                result = dynamoDB.batchGetItem(req);
                resultantItems.addAll(result.getResponses().get(tableName));
                req.setRequestItems(result.getUnprocessedKeys());
            } catch (AmazonClientException e) {
                throw this.convertDynamoDBException(e, "batch get", null /*no conditions for reads*/);
            }
        } while (false == result.getUnprocessedKeys().isEmpty());
    });

    return resultantItems.stream().map(legacyItem -> Item.fromMap(InternalUtils.toSimpleMapValue(legacyItem)))
            .map(item -> convertItemToDomain(item)).collect(Collectors.toList());
}

---

From source file:ei.ne.ke.cassandra.cql3.AstyanaxCql3Repository.java

/**
 * {@inheritDoc}/*from w w w .jav  a2s  .  c om*/
 */
@Override
public synchronized <S extends T> List<S> save(Iterable<S> entities) {
    List<Callable<List<S>>> todo = Lists.newArrayListWithExpectedSize(Iterables.size(entities) / batchSize);
    for (Iterable<S> partition : Iterables.partition(entities, batchSize)) {
        todo.add(new Saver<S>(partition));
    }
    try {
        List<Future<List<S>>> futureResults = executorService.invokeAll(todo);
        waitUntilCompletion(futureResults);
    } catch (InterruptedException e) {
        Thread.interrupted();
    }
    return (List<S>) entities;
}

---

From source file:org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStoreJDBC.java

public List<RDBRow> read(Connection connection, RDBTableMetaData tmd, Collection<String> allKeys)
        throws SQLException {

    List<RDBRow> rows = new ArrayList<RDBRow>();

    for (List<String> keys : Iterables.partition(allKeys, RDBJDBCTools.MAX_IN_CLAUSE)) {
        PreparedStatementComponent inClause = RDBJDBCTools.createInStatement("ID", keys, tmd.isIdBinary());
        StringBuilder query = new StringBuilder();
        query.append("select ID, MODIFIED, MODCOUNT, CMODCOUNT, HASBINARY, DELETEDONCE, DATA, BDATA from ");
        query.append(tmd.getName());//from ww  w  .ja  v  a2s.c  om
        query.append(" where ").append(inClause.getStatementComponent());

        PreparedStatement stmt = connection.prepareStatement(query.toString());
        stmt.setPoolable(false);
        try {
            inClause.setParameters(stmt, 1);
            ResultSet rs = stmt.executeQuery();

            while (rs.next()) {
                int col = 1;
                String id = getIdFromRS(tmd, rs, col++);
                long modified = readLongFromResultSet(rs, col++);
                long modcount = readLongFromResultSet(rs, col++);
                long cmodcount = readLongFromResultSet(rs, col++);
                long hasBinary = rs.getLong(col++);
                long deletedOnce = rs.getLong(col++);
                String data = rs.getString(col++);
                byte[] bdata = rs.getBytes(col++);
                RDBRow row = new RDBRow(id, hasBinary == 1, deletedOnce == 1, modified, modcount, cmodcount,
                        data, bdata);
                rows.add(row);
            }
        } catch (SQLException ex) {
            LOG.debug("attempting to read " + keys, ex);
            // DB2 throws an SQLException for invalid keys; handle this more
            // gracefully
            if ("22001".equals(ex.getSQLState())) {
                try {
                    connection.rollback();
                } catch (SQLException ex2) {
                    LOG.debug("failed to rollback", ex2);
                }
                return null;
            } else {
                throw (ex);
            }
        } finally {
            stmt.close();
        }
    }
    return rows;
}

---

From source file:ome.services.graphs.GraphTraversal.java

/**
 * For the given class name and IDs, construct the corresponding {@link CI} instances without loading the persisted objects,
 * and ensure that their {@link ome.model.internal.Details} are noted.
 * @param className a model object class name
 * @param ids IDs of instances of the class
 * @return the {@link CI} instances indexed by object ID
 * @throws GraphException if an object could not be converted to an unloaded instance
 *//*from  w w w.  java2  s  .co  m*/
private Map<Long, CI> findObjectDetails(String className, Collection<Long> ids) throws GraphException {
    final Map<Long, CI> objectsById = new HashMap<Long, CI>();
    final Set<Long> idsToQuery = new HashSet<Long>();

    for (final Long id : ids) {
        final CI object = new CI(className, id);
        final CI alias = planning.aliases.get(object);
        if (alias == null) {
            idsToQuery.add(id);
        } else {
            objectsById.put(object.id, alias);
        }
    }

    if (!idsToQuery.isEmpty()) {
        /* query persisted object instances without loading them */
        final String rootQuery = "FROM " + className + " WHERE id IN (:ids)";
        for (final List<Long> idsBatch : Iterables.partition(idsToQuery, BATCH_SIZE)) {
            final Iterator<Object> objectInstances = session.createQuery(rootQuery)
                    .setParameterList("ids", idsBatch).iterate();
            while (objectInstances.hasNext()) {
                /*final*/ Object objectInstance = objectInstances.next();
                if (objectInstance instanceof HibernateProxy) {
                    /* TODO: this is an awkward hack pending Hibernate 4's type() function */
                    final LazyInitializer initializer = ((HibernateProxy) objectInstance)
                            .getHibernateLazyInitializer();
                    final Long id = (Long) initializer.getIdentifier();
                    String realClassName = initializer.getEntityName();
                    boolean lookForSubclass = true;
                    while (lookForSubclass) {
                        lookForSubclass = false;
                        for (final String subclassName : model.getSubclassesOf(realClassName)) {
                            final String classQuery = "FROM " + subclassName + " WHERE id = :id";
                            final Iterator<Object> instance = session.createQuery(classQuery)
                                    .setParameter("id", id).iterate();
                            if (instance.hasNext()) {
                                realClassName = subclassName;
                                lookForSubclass = true;
                                break;
                            }
                        }
                    }
                    objectInstance = new CI(realClassName, id).toIObject();
                }
                final CI object = new CI((IObject) objectInstance);
                objectsById.put(object.id, object);
                planning.aliases.put(new CI(className, object.id), object);
            }
        }

        /* construct query according to which details may be queried */
        final Set<String> linkProperties = new HashSet<String>();
        for (final String superclassName : model.getSuperclassesOfReflexive(className)) {
            final Set<Entry<String, String>> forwardLinks = model.getLinkedTo(superclassName);
            for (final Entry<String, String> forwardLink : forwardLinks) {
                linkProperties.add(forwardLink.getValue());
            }
        }
        final List<String> soughtProperties = ImmutableList.of("details.owner", "details.group");
        final List<String> selectTerms = new ArrayList<String>(soughtProperties.size() + 1);
        selectTerms.add("root.id");
        for (final String soughtProperty : soughtProperties) {
            if (linkProperties.contains(soughtProperty)) {
                selectTerms.add("root." + soughtProperty);
            } else {
                selectTerms.add("NULLIF(0,0)"); /* a simple NULL doesn't work in Hibernate 3.5 */
            }
        }
        selectTerms.add(
                "root.details.permissions"); /* to include among soughtProperties once GraphPathBean knows of it */
        final String detailsQuery = "SELECT " + Joiner.on(',').join(selectTerms) + " FROM " + className
                + " AS root WHERE root.id IN (:ids)";

        /* query and note details of objects */
        for (final List<Long> idsBatch : Iterables.partition(idsToQuery, BATCH_SIZE)) {
            final Query hibernateQuery = session.createQuery(detailsQuery).setParameterList("ids", idsBatch);
            for (final Object[] result : (List<Object[]>) hibernateQuery.list()) {
                final ome.model.internal.Details details = ome.model.internal.Details.create();
                final Long id = (Long) result[0];
                details.setOwner((Experimenter) result[1]);
                details.setGroup((ExperimenterGroup) result[2]);
                details.setPermissions((Permissions) result[3]);
                noteDetails(objectsById.get(id), details);
            }
        }
    }

    return objectsById;
}

---

From source file:com.querydsl.sql.SQLSerializer.java

@SuppressWarnings("unchecked")
@Override/*  w w w. ja v a  2  s  .c  o m*/
protected void visitOperation(Class<?> type, Operator operator, List<? extends Expression<?>> args) {
    boolean pathAdded = false;
    if (args.size() == 2 && !useLiterals && args.get(0) instanceof Path<?> && args.get(1) instanceof Constant<?>
            && operator != Ops.NUMCAST) {
        Object constant = ((Constant<?>) args.get(1)).getConstant();
        if (!Collection.class.isInstance(constant) || !((Collection) constant).isEmpty()) {
            for (Element element : templates.getTemplate(operator).getElements()) {
                if (element instanceof Template.ByIndex && ((Template.ByIndex) element).getIndex() == 1) {
                    constantPaths.add((Path<?>) args.get(0));
                    pathAdded = true;
                    break;
                }
            }
        }
    }

    if (operator == SQLOps.UNION || operator == SQLOps.UNION_ALL) {
        boolean oldUnion = inUnion;
        inUnion = true;
        super.visitOperation(type, operator, args);
        inUnion = oldUnion;

    } else if (operator == Ops.LIKE && args.get(1) instanceof Constant<?>) {
        final String escape = String.valueOf(templates.getEscapeChar());
        final String escaped = args.get(1).toString().replace(escape, escape + escape);
        super.visitOperation(String.class, Ops.LIKE,
                ImmutableList.of(args.get(0), ConstantImpl.create(escaped)));

    } else if (operator == Ops.STRING_CAST) {
        final String typeName = configuration.getTypeNameForCast(String.class);
        super.visitOperation(String.class, SQLOps.CAST,
                ImmutableList.of(args.get(0), ConstantImpl.create(typeName)));

    } else if (operator == Ops.NUMCAST) {
        @SuppressWarnings("unchecked") //this is the second argument's type
        Constant<Class<?>> expectedConstant = (Constant<Class<?>>) args.get(1);

        final Class<?> targetType = expectedConstant.getConstant();
        final String typeName = configuration.getTypeNameForCast(targetType);
        super.visitOperation(targetType, SQLOps.CAST,
                ImmutableList.of(args.get(0), ConstantImpl.create(typeName)));

    } else if (operator == Ops.ALIAS) {
        if (stage == Stage.SELECT || stage == Stage.FROM) {
            if (args.get(1) instanceof Path && !((Path<?>) args.get(1)).getMetadata().isRoot()) {
                Path<?> path = (Path<?>) args.get(1);
                args = ImmutableList.of(args.get(0),
                        ExpressionUtils.path(path.getType(), path.getMetadata().getName()));
            }
            super.visitOperation(type, operator, args);
        } else {
            // handle only target
            handle(args.get(1));
        }

    } else if ((operator == Ops.IN || operator == Ops.NOT_IN) && args.get(0) instanceof Path<?>
            && args.get(1) instanceof Constant<?>) {
        //The type of the constant expression is compatible with the left
        //expression, since the compile time checking mandates it to be.
        @SuppressWarnings("unchecked")
        Collection<Object> coll = ((Constant<Collection<Object>>) args.get(1)).getConstant();
        if (coll.isEmpty()) {
            super.visitOperation(type, operator == Ops.IN ? Ops.EQ : Ops.NE,
                    ImmutableList.of(Expressions.ONE, Expressions.TWO));
        } else {
            if (templates.getListMaxSize() == 0 || coll.size() <= templates.getListMaxSize()) {
                super.visitOperation(type, operator, args);
            } else {
                //The type of the path is compatible with the constant
                //expression, since the compile time checking mandates it to be
                @SuppressWarnings("unchecked")
                Expression<Object> path = (Expression<Object>) args.get(0);
                if (pathAdded) {
                    constantPaths.removeLast();
                }
                Iterable<List<Object>> partitioned = Iterables.partition(coll, templates.getListMaxSize());
                Predicate result;
                if (operator == Ops.IN) {
                    result = ExpressionUtils.inAny(path, partitioned);
                } else {
                    result = ExpressionUtils.notInAny(path, partitioned);
                }
                result.accept(this, null);
            }
        }

    } else if (operator == SQLOps.WITH_COLUMNS) {
        boolean oldSkipParent = skipParent;
        skipParent = true;
        super.visitOperation(type, operator, args);
        skipParent = oldSkipParent;

    } else if (operator == Ops.ORDER) {
        List<OrderSpecifier<?>> order = ((Constant<List<OrderSpecifier<?>>>) args.get(0)).getConstant();
        handleOrderBy(order);

    } else {
        super.visitOperation(type, operator, args);
    }
}

---

From source file:edu.umn.msi.tropix.persistence.dao.hibernate.TropixObjectDaoImpl.java

public Multimap<String, String> getRoles(final String userIdentity, final Iterable<String> objectIds) {
    final Multimap<String, String> roleMap = HashMultimap.create();
    final Iterable<List<String>> objectIdPartitions = Iterables.partition(objectIds, 500);
    for (List<String> objectIdPartition : objectIdPartitions) {
        final Query query = getSession().getNamedQuery("getRolesForObjects");
        query.setParameter("userId", userIdentity);
        query.setParameterList("objectIds", objectIdPartition);
        @SuppressWarnings("unchecked")
        final List<Object[]> results = query.list();
        for (Object[] result : results) {
            roleMap.put((String) result[0], (String) result[1]);
        }//from  w  w  w .j a  va 2s  .c  o m
    }
    return roleMap;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• »