Example usage for com.mongodb.client FindIterable batchSize

Introduction

In this page you can find the example usage for com.mongodb.client FindIterable batchSize.

Prototype

@Override
FindIterable<TResult> batchSize(int batchSize);

Source Link

Document

Sets the number of documents to return per batch.

Usage

From source file:com.facebook.presto.mongodb.MongoSession.java

License:Apache License

public MongoCursor<Document> execute(MongoSplit split, List<MongoColumnHandle> columns) {
    Document output = new Document();
    for (MongoColumnHandle column : columns) {
        output.append(column.getName(), 1);
    }/*  ww  w .  ja  v a 2 s  . com*/
    MongoCollection<Document> collection = getCollection(split.getSchemaTableName());
    FindIterable<Document> iterable = collection.find(buildQuery(split.getTupleDomain())).projection(output);

    if (cursorBatchSize != 0) {
        iterable.batchSize(cursorBatchSize);
    }

    return iterable.iterator();
}

---

From source file:com.jaeksoft.searchlib.crawler.database.DatabaseCrawlMongoDbThread.java

License:Open Source License

final private void runner_update(FindIterable<Document> iterable)
        throws SearchLibException, ClassNotFoundException, InstantiationException, IllegalAccessException,
        IOException, ParseException, SyntaxError, URISyntaxException, InterruptedException {
    final int limit = databaseCrawl.getBufferSize();
    iterable.batchSize(limit);
    DatabaseFieldMap databaseFieldMap = databaseCrawl.getFieldMap();
    List<IndexDocument> indexDocumentList = new ArrayList<IndexDocument>(0);
    LanguageEnum lang = databaseCrawl.getLang();
    FieldMapContext fieldMapContext = new FieldMapContext(client, lang);
    String uniqueField = client.getSchema().getUniqueField();
    MongoCursor<Document> cursor = iterable.iterator();
    while (cursor.hasNext() && !isAborted()) {

        String json = JSON.serialize(cursor.next());
        Object document = Configuration.defaultConfiguration().jsonProvider().parse(json);
        IndexDocument indexDocument = new IndexDocument(lang);
        databaseFieldMap.mapJson(fieldMapContext, document, indexDocument);
        if (uniqueField != null && !indexDocument.hasContent(uniqueField)) {
            rwl.w.lock();/*from  w w  w. ja  v  a2s  .  com*/
            try {
                ignoredDocumentCount++;
            } finally {
                rwl.w.unlock();
            }
            continue;
        }
        indexDocumentList.add(indexDocument);
        rwl.w.lock();
        try {
            pendingIndexDocumentCount++;
        } finally {
            rwl.w.unlock();
        }
        if (index(indexDocumentList, limit))
            setStatus(CrawlStatus.CRAWL);

    }
    index(indexDocumentList, 0);
}

---

From source file:io.dirigible.mongodb.jdbc.MongodbStatement.java

License:Apache License

/**
 * Input string: the document specification as defined in https://docs.mongodb.org/manual/reference/command/find/#dbcmd.find
 *///  w  w  w. j a  v a  2 s .c  o m
@Override
public ResultSet executeQuery(String sql) throws SQLException {
    MongoDatabase db = this.conn.getMongoDb();
    BsonDocument filterDocument = null;
    if (sql == null || sql.length() < 1)//that is a call to find() in terms of mongodb queries
        filterDocument = new BsonDocument();
    else
        filterDocument = BsonDocument.parse(sql);
    /*
     *  TODO: With 3.2 use https://docs.mongodb.org/manual/reference/command/find/#dbcmd.find instead
     *    Document response = this.conn.getMongoDb().runCommand(filterDocument); //MongoDB 3.2 only. Won't work on 3.0
     */
    String collectionName = filterDocument.containsKey("find") ? filterDocument.getString("find").getValue()
            : null;
    if (collectionName == null)
        collectionName = this.conn.getCollectionName();//fallback if any
    if (collectionName == null)
        throw new IllegalArgumentException("Specifying a collection is mandatory for query operations");
    BsonDocument filter = filterDocument.containsKey("filter") ? filterDocument.getDocument("filter") : null;

    FindIterable<Document> searchHits = null;
    if (filter == null)
        searchHits = db.getCollection(collectionName).find();
    else
        searchHits = db.getCollection(collectionName).find(filter);
    if (filterDocument.containsKey("batchSize"))
        searchHits.batchSize(filterDocument.getInt32("batchSize").getValue());
    if (filterDocument.containsKey("limit"))
        searchHits.limit(filterDocument.getInt32("limit").getValue());
    if (filterDocument.containsKey("sort"))
        searchHits.sort(filterDocument.getDocument("sort"));
    return new MongodbResultSet(this, searchHits);
}

---

From source file:org.apache.nifi.processors.mongodb.GetMongo.java

License:Apache License

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final ComponentLog logger = getLogger();

    final Document query = context.getProperty(QUERY).isSet()
            ? Document.parse(context.getProperty(QUERY).evaluateAttributeExpressions().getValue())
            : null;/*  w ww.  j  a  va2  s  . c  om*/
    final Document projection = context.getProperty(PROJECTION).isSet()
            ? Document.parse(context.getProperty(PROJECTION).evaluateAttributeExpressions().getValue())
            : null;
    final Document sort = context.getProperty(SORT).isSet()
            ? Document.parse(context.getProperty(SORT).evaluateAttributeExpressions().getValue())
            : null;
    final String jsonTypeSetting = context.getProperty(JSON_TYPE).getValue();
    configureMapper(jsonTypeSetting);

    final MongoCollection<Document> collection = getCollection(context);

    try {
        final FindIterable<Document> it = query != null ? collection.find(query) : collection.find();
        if (projection != null) {
            it.projection(projection);
        }
        if (sort != null) {
            it.sort(sort);
        }
        if (context.getProperty(LIMIT).isSet()) {
            it.limit(context.getProperty(LIMIT).evaluateAttributeExpressions().asInteger());
        }
        if (context.getProperty(BATCH_SIZE).isSet()) {
            it.batchSize(context.getProperty(BATCH_SIZE).evaluateAttributeExpressions().asInteger());
        }

        final MongoCursor<Document> cursor = it.iterator();
        ComponentLog log = getLogger();
        try {
            FlowFile flowFile = null;
            if (context.getProperty(RESULTS_PER_FLOWFILE).isSet()) {
                int ceiling = context.getProperty(RESULTS_PER_FLOWFILE).evaluateAttributeExpressions()
                        .asInteger();
                List<Document> batch = new ArrayList<>();

                while (cursor.hasNext()) {
                    batch.add(cursor.next());
                    if (batch.size() == ceiling) {
                        try {
                            if (log.isDebugEnabled()) {
                                log.debug("Writing batch...");
                            }
                            String payload = buildBatch(batch, jsonTypeSetting);
                            writeBatch(payload, context, session);
                            batch = new ArrayList<>();
                        } catch (IOException ex) {
                            getLogger().error("Error building batch", ex);
                        }
                    }
                }
                if (batch.size() > 0) {
                    try {
                        writeBatch(buildBatch(batch, jsonTypeSetting), context, session);
                    } catch (IOException ex) {
                        getLogger().error("Error sending remainder of batch", ex);
                    }
                }
            } else {
                while (cursor.hasNext()) {
                    flowFile = session.create();
                    flowFile = session.write(flowFile, new OutputStreamCallback() {
                        @Override
                        public void process(OutputStream out) throws IOException {
                            String json;
                            if (jsonTypeSetting.equals(JSON_TYPE_STANDARD)) {
                                json = mapper.writerWithDefaultPrettyPrinter()
                                        .writeValueAsString(cursor.next());
                            } else {
                                json = cursor.next().toJson();
                            }
                            IOUtils.write(json, out);
                        }
                    });
                    flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(),
                            "application/json");

                    session.getProvenanceReporter().receive(flowFile, getURI(context));
                    session.transfer(flowFile, REL_SUCCESS);
                }
            }

            session.commit();

        } finally {
            cursor.close();
        }

    } catch (final RuntimeException e) {
        context.yield();
        session.rollback();
        logger.error("Failed to execute query {} due to {}", new Object[] { query, e }, e);
    }
}

---

From source file:org.axonframework.mongo.eventsourcing.eventstore.AbstractMongoEventStorageStrategy.java

License:Apache License

@Override
public List<? extends DomainEventData<?>> findDomainEvents(MongoCollection<Document> collection,
        String aggregateIdentifier, long firstSequenceNumber, int batchSize) {
    FindIterable<Document> cursor = collection
            .find(and(eq(eventConfiguration.aggregateIdentifierProperty(), aggregateIdentifier),
                    gte(eventConfiguration.sequenceNumberProperty(), firstSequenceNumber)))
            .sort(new BasicDBObject(eventConfiguration().sequenceNumberProperty(), ORDER_ASC));
    cursor = cursor.batchSize(batchSize);
    return stream(cursor.spliterator(), false).flatMap(this::extractEvents)
            .filter(event -> event.getSequenceNumber() >= firstSequenceNumber).collect(Collectors.toList());
}

---

From source file:org.axonframework.mongo.eventsourcing.eventstore.documentperevent.DocumentPerEventStorageStrategy.java

License:Apache License

@Override
protected FindIterable<Document> applyBatchSize(FindIterable<Document> cursor, int batchSize) {
    return cursor.batchSize(batchSize);
}

---

From source file:org.lambda3.indra.core.impl.MongoVectorSpace.java

License:Open Source License

private void collectVectors(Set<String> terms, int limit) {
    Set<String> toFetch = terms.stream().filter(t -> !this.vectorsCache.containsKey(t))
            .collect(Collectors.toSet());

    logger.debug("Cache has {} vectors, need to fetch more {}", terms.size() - toFetch.size(), toFetch.size());

    if (!toFetch.isEmpty()) {
        logger.info("Collecting {} term vectors from {}", toFetch.size(), dbName);
        FindIterable<Document> docs = getColl().find(Filters.in(termFieldName, toFetch));
        if (docs != null) {
            docs.batchSize(toFetch.size());
            for (Document doc : docs) {
                this.vectorsCache.put(doc.getString(termFieldName), unmarshall(doc, limit));
            }/*w w w  .  jav a2  s  . co m*/
        }
    }
}

---

From source file:org.opencb.commons.datastore.mongodb.MongoDBNativeQuery.java

License:Apache License

public FindIterable<Document> find(Bson query, Bson projection, QueryOptions options) {

    if (projection == null) {
        projection = getProjection(projection, options);
    }/*ww  w .j  a  v  a  2s. c o  m*/

    FindIterable<Document> findIterable = dbCollection.find(query).projection(projection);

    int limit = (options != null) ? options.getInt(QueryOptions.LIMIT, 0) : 0;
    if (limit > 0) {
        findIterable.limit(limit);
    }

    int skip = (options != null) ? options.getInt(QueryOptions.SKIP, 0) : 0;
    if (skip > 0) {
        findIterable.skip(skip);
    }

    Object sortObject = (options != null) ? options.get(QueryOptions.SORT) : null;
    if (sortObject != null) {
        if (sortObject instanceof Bson) {
            findIterable.sort(((Bson) sortObject));
        } else if (sortObject instanceof String) {
            String order = options.getString(QueryOptions.ORDER, "DESC");
            if (order.equalsIgnoreCase(QueryOptions.ASCENDING) || order.equalsIgnoreCase("ASC")
                    || order.equals("1")) {
                findIterable.sort(Sorts.ascending(((String) sortObject)));
            } else {
                findIterable.sort(Sorts.descending(((String) sortObject)));
            }
        }
    }

    if (options != null && options.containsKey(MongoDBCollection.BATCH_SIZE)) {
        findIterable.batchSize(options.getInt(MongoDBCollection.BATCH_SIZE, 20));
    }

    if (options != null && options.containsKey(QueryOptions.TIMEOUT)) {
        findIterable.maxTime(options.getLong(QueryOptions.TIMEOUT), TimeUnit.MILLISECONDS);
    }

    return findIterable;
}

---

From source file:org.opencb.commons.datastore.mongodb.MongoPersistentCursor.java

License:Apache License

protected MongoPersistentCursor resume(Object lastObjectId) {
    Bson query;//from  w  w  w.  j  ava2s.c  o  m
    if (lastObjectId != null) {
        query = Filters.and(Filters.gt("_id", lastObjectId), this.query);
    } else {
        query = this.query;
    }
    FindIterable<Document> iterable = newFindIterable(query, this.projection, this.options);
    if (!options.containsKey(QueryOptions.SORT)) {
        iterable.sort(Sorts.ascending("$natural"));
    }
    mongoCursor = iterable.batchSize(batchSize).limit(limit).skip(skip).iterator();
    return this;
}

---

From source file:org.opencb.opencga.storage.mongodb.variant.adaptors.VariantMongoDBIterator.java

License:Apache License

VariantMongoDBIterator(FindIterable<Document> dbCursor, DocumentToVariantConverter documentToVariantConverter,
        int batchSize) {
    this.documentToVariantConverter = documentToVariantConverter;
    if (batchSize > 0) {
        dbCursor.batchSize(batchSize);
    }/*from www .  j a v  a  2s  .c  o m*/
    this.dbCursor = dbCursor.iterator();
}

---