Example usage for com.mongodb BasicDBObject get

List of usage examples for com.mongodb BasicDBObject get

  1. HOME
  2. Java
  3. com.mongodb
  4. com.mongodb.*
  5. BasicDBObject
  6. get

Introduction

In this page you can find the example usage for com.mongodb BasicDBObject get.

Prototype

public Object get(final String key)

Source Link

Document

Gets a value from this object

Usage

From source file:com.ibm.bluemix.smartveggie.dao.SubOutletVendorAllocationDaoImpl.java

@Override
public BasicDBObject getVendorForSubOutlet(String vendorUserName) {
    try {//from   w w w  . j  a v a2s.c  o m
        System.out.println("starting object retrieve..");
        DB db = MongodbConnection.getMongoDB();
        BasicDBObject query = new BasicDBObject();
        query.append("vendorUsername", vendorUserName);
        System.out.println("Querying for getting vendor suboutlet: " + query);
        DBCollection col = db.getCollection(ICollectionName.COLLECTION_ALLOC_SUBOUTLETS);
        DBCursor cursor = col.find(query);
        BasicDBObject obj = null;
        while (cursor.hasNext()) {
            obj = (BasicDBObject) cursor.next();
            //Check the date
            Date currentDate = Calendar.getInstance().getTime();
            SimpleDateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
            String allocationTillString = (String) obj.get("suboutletAllocatedTo");
            if (allocationTillString != null) {
                Date allocEndDate = null;
                try {
                    allocEndDate = dateFormat.parse(allocationTillString);
                    if (allocEndDate.before(currentDate)) {
                        System.out.println("Suboutlet Allocation already ended....");
                        //subOutletAvailableList.add(allocation);
                    } else {
                        break;
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
                System.out.println("Retrieved Allocated Vendor suboutlet: " + obj);
            }
        }
        cursor.close();
        return obj;
    } catch (Exception e) {
        throw e;
    }
}

From source file:com.ibm.ws.lars.rest.PersistenceBean.java

License:Apache License

/**
 * @param attachment//from   w  ww  .  j a  v a 2s .co  m
 * @return
 */
@Override
public Attachment createAttachmentMetadata(Attachment attachment) {
    BasicDBObject state = new BasicDBObject(attachment.getProperties());
    convertHexIdToObjectId(state);

    DBCollection coll = getAttachmentCollection();
    if (logger.isLoggable(Level.FINE)) {
        logger.fine("createAttachmentMetadata: inserting new attachment " + state);
    }
    coll.insert(state);
    Object idObject = state.get(ID);
    String id;
    if (idObject instanceof String) {
        id = (String) idObject;
    } else if (idObject instanceof ObjectId) {
        id = ((ObjectId) idObject).toStringMongod();
    } else {
        throw new AssertionError("_id should be either String of ObjectId");
    }
    try {
        return retrieveAttachmentMetadata(id);
    } catch (NonExistentArtefactException e) {
        throw new RepositoryException("Created attachment could not be retrieved from the persistence store",
                e);
    }
}

From source file:com.ikanow.aleph2.shared.crud.mongodb.utils.MongoDbUtils.java

License:Apache License

/** Inserts an object into field1.field2, creating objects along the way
 * @param mutable the mutable object into which the the nested field is inserted
 * @param parent the top level fieldname
 * @param nested the nested fieldname /*from  www.  j  a va2s  . co m*/
 * @param to_insert the object to insert
 */
protected static void nestedPut(final BasicDBObject mutable, final String parent, final String nested,
        final Object to_insert) {
    final DBObject dbo = (DBObject) mutable.get(parent);
    if (null != dbo) {
        dbo.put(nested, to_insert);
    } else {
        BasicDBObject new_dbo = new BasicDBObject();
        new_dbo.put(nested, to_insert);
        mutable.put(parent, new_dbo);
    }
}

From source file:com.ikanow.aleph2.v1.document_db.utils.LegacyV1HadoopUtils.java

License:Open Source License

/** parse the V1 query string 
 * @param query/*from ww w  . jav a2s . co m*/
 * @return the required objects embedded in various tuples
 */
public static Tuple4<String, Tuple2<Integer, Integer>, BasicDBObject, DBObject> parseQueryObject(
        final String query, final List<String> community_ids) {
    // Some fixed variables just to avoid changing the guts of the (tested in v1) code
    final boolean isCustomTable = false;
    @SuppressWarnings("unused")
    Integer nDebugLimit = null;
    final boolean bLocalMode = false;
    @SuppressWarnings("unused")
    final Boolean incrementalMode = null;
    final String input = "doc_metadata.metadata";

    // Output objects
    final String out_query;
    int nSplits = 8;
    int nDocsPerSplit = 12500;

    List<ObjectId> communityIds = community_ids.stream().map(s -> new ObjectId(s)).collect(Collectors.toList());

    //C/P code:

    //add communities to query if this is not a custom table
    BasicDBObject oldQueryObj = null;
    BasicDBObject srcTags = null;
    // Start with the old query:
    if (query.startsWith("{")) {
        oldQueryObj = (BasicDBObject) com.mongodb.util.JSON.parse(query);
    } else {
        oldQueryObj = new BasicDBObject();
    }
    boolean elasticsearchQuery = oldQueryObj.containsField("qt") && !isCustomTable;
    @SuppressWarnings("unused")
    int nLimit = 0;
    if (oldQueryObj.containsField(":limit")) {
        nLimit = oldQueryObj.getInt(":limit");
        oldQueryObj.remove(":limit");
    }
    if (oldQueryObj.containsField(":splits")) {
        nSplits = oldQueryObj.getInt(":splits");
        oldQueryObj.remove(":splits");
    }
    if (oldQueryObj.containsField(":srctags")) {
        srcTags = new BasicDBObject(SourcePojo.tags_, oldQueryObj.get(":srctags"));
        oldQueryObj.remove(":srctags");
    }
    if (bLocalMode) { // If in local mode, then set this to a large number so we always run inside our limit/split version
        // (since for some reason MongoInputFormat seems to fail on large collections)
        nSplits = InfiniteMongoSplitter.MAX_SPLITS;
    }
    if (oldQueryObj.containsField(":docsPerSplit")) {
        nDocsPerSplit = oldQueryObj.getInt(":docsPerSplit");
        oldQueryObj.remove(":docsPerSplit");
    }
    final DBObject fields = (DBObject) oldQueryObj.remove(":fields");
    oldQueryObj.remove(":output");
    oldQueryObj.remove(":reducers");
    @SuppressWarnings("unused")
    String mapperKeyClass = oldQueryObj.getString(":mapper_key_class", "");
    @SuppressWarnings("unused")
    String mapperValueClass = oldQueryObj.getString(":mapper_value_class", "");
    oldQueryObj.remove(":mapper_key_class");
    oldQueryObj.remove(":mapper_value_class");
    String cacheList = null;
    Object cacheObj = oldQueryObj.get(":caches");
    if (null != cacheObj) {
        cacheList = cacheObj.toString(); // (either array of strings, or single string)
        if (!cacheList.startsWith("[")) {
            cacheList = "[" + cacheList + "]"; // ("must" now be valid array)
        }
        oldQueryObj.remove(":caches");
    } //TESTED

    //      if (null != nDebugLimit) { // (debug mode override)
    //         nLimit = nDebugLimit;
    //      }
    //      boolean tmpIncMode = ( null != incrementalMode) && incrementalMode; 

    @SuppressWarnings("unused")
    String otherCollections = null;
    Date fromOverride = null;
    Date toOverride = null;
    Object fromOverrideObj = oldQueryObj.remove(":tmin");
    Object toOverrideObj = oldQueryObj.remove(":tmax");
    if (null != fromOverrideObj) {
        fromOverride = dateStringFromObject(fromOverrideObj, true);
    }
    if (null != toOverrideObj) {
        toOverride = dateStringFromObject(toOverrideObj, false);
    }

    if (!isCustomTable) {
        if (elasticsearchQuery) {
            oldQueryObj.put("communityIds", communityIds);
            //tmin/tmax not supported - already have that capability as part of the query
        } else {
            if (input.equals("feature.temporal")) {
                if ((null != fromOverride) || (null != toOverride)) {
                    oldQueryObj.put("value.maxTime", createDateRange(fromOverride, toOverride, true));
                } //TESTED
                oldQueryObj.put("_id.c", new BasicDBObject(DbManager.in_, communityIds));
            } else {
                oldQueryObj.put(DocumentPojo.communityId_, new BasicDBObject(DbManager.in_, communityIds));
                if ((null != fromOverride) || (null != toOverride)) {
                    oldQueryObj.put(JsonUtils._ID, createDateRange(fromOverride, toOverride, false));
                } //TESTED         
                if (input.equals("doc_metadata.metadata")) {
                    oldQueryObj.put(DocumentPojo.index_, new BasicDBObject(DbManager.ne_, "?DEL?")); // (ensures not soft-deleted)
                }
            }
        }
    } else {
        throw new RuntimeException("Custom Tables not currently supported (no plans to)");
        //         if ((null != fromOverride) || (null != toOverride)) {
        //            oldQueryObj.put(JsonUtils._ID, createDateRange(fromOverride, toOverride, false));
        //         }//TESTED
        //         //get the custom table (and database)
        //
        //         String[] candidateInputs = input.split("\\s*,\\s*");
        //         input = CustomOutputManager.getCustomDbAndCollection(candidateInputs[0]);
        //         if (candidateInputs.length > 1) {            
        //            otherCollections = Arrays.stream(candidateInputs)
        //                  .skip(1L)
        //                  .map(i -> CustomOutputManager.getCustomDbAndCollection(i))
        //                  .map(i -> "mongodb://"+dbserver+"/"+i).collect(Collectors.joining("|"));
        //         }
    }
    out_query = oldQueryObj.toString();

    return Tuples._4T(out_query, Tuples._2T(nSplits, nDocsPerSplit), srcTags, fields);
}

From source file:com.ikanow.infinit.e.api.config.source.SourceHandler.java

License:Open Source License

/**
 * testSource/* www  .jav a 2 s  .co m*/
 * @param sourceJson
 * @param nNumDocsToReturn
 * @param bReturnFullText
 * @param userIdStr
 * @return
 */
public ResponsePojo testSource(String sourceJson, int nNumDocsToReturn, boolean bReturnFullText,
        boolean bRealDedup, String userIdStr) {
    ResponsePojo rp = new ResponsePojo();
    try {
        SourcePojo source = null;
        SourcePojoSubstitutionApiMap apiMap = new SourcePojoSubstitutionApiMap(new ObjectId(userIdStr));
        try {
            source = ApiManager.mapFromApi(sourceJson, SourcePojo.class, apiMap);
            source.fillInSourcePipelineFields();
        } catch (Exception e) {
            rp.setResponse(new ResponseObject("Test Source", false,
                    "Error deserializing source (JSON is valid but does not match schema): " + e.getMessage()));
            return rp;
        }
        if (null == source.getKey()) {
            source.setKey(source.generateSourceKey()); // (a dummy value, not guaranteed to be unique)
        }
        if ((null == source.getExtractType()) || !source.getExtractType().equals("Federated")) {
            String testUrl = source.getRepresentativeUrl();
            if (null == testUrl) {
                rp.setResponse(
                        new ResponseObject("Test Source", false, "Error, source contains no URL to harvest"));
                return rp;
            }
        }
        if (null == source.getTags()) {
            source.setTags(new HashSet<String>());
        }

        // This is the only field that you don't normally need to specify in save but will cause 
        // problems if it's not populated in test.
        ObjectId userId = new ObjectId(userIdStr);
        // Set owner (overwrite, for security reasons)
        source.setOwnerId(userId);
        if (null == source.getCommunityIds()) {
            source.setCommunityIds(new TreeSet<ObjectId>());
        }
        if (!source.getCommunityIds().isEmpty()) { // need to check that I'm allowed the specified community...
            if ((1 == source.getCommunityIds().size())
                    && (userId.equals(source.getCommunityIds().iterator().next()))) {
                // we're OK only community id is user community
            } //TESTED
            else {
                HashSet<ObjectId> communities = SocialUtils.getUserCommunities(userIdStr);
                Iterator<ObjectId> it = source.getCommunityIds().iterator();
                while (it.hasNext()) {
                    ObjectId src = it.next();
                    if (!communities.contains(src)) {
                        rp.setResponse(new ResponseObject("Test Source", false,
                                "Authentication error: you don't belong to this community: " + src));
                        return rp;
                    } //TESTED
                }
            } //TESTED
        }
        // Always add the userId to the source community Id (so harvesters can tell if they're running in test mode or not...) 
        source.addToCommunityIds(userId); // (ie user's personal community, always has same _id - not that it matters)

        // Check the source's admin status
        source.setOwnedByAdmin(RESTTools.adminLookup(userId.toString(), false));

        if (bRealDedup) { // Want to test update code, so ignore update cycle
            if (null != source.getRssConfig()) {
                source.getRssConfig().setUpdateCycle_secs(1); // always update
            }
        }
        HarvestController harvester = new HarvestController(true);
        if (nNumDocsToReturn > 100) { // (seems reasonable)
            nNumDocsToReturn = 100;
        }
        harvester.setStandaloneMode(nNumDocsToReturn, bRealDedup);
        List<DocumentPojo> toAdd = new LinkedList<DocumentPojo>();
        List<DocumentPojo> toUpdate = new LinkedList<DocumentPojo>();
        List<DocumentPojo> toRemove = new LinkedList<DocumentPojo>();
        if (null == source.getHarvestStatus()) {
            source.setHarvestStatus(new SourceHarvestStatusPojo());
        }
        String oldMessage = source.getHarvestStatus().getHarvest_message();
        // SPECIAL CASE: FOR FEDERATED QUERIES
        if ((null != source.getExtractType()) && source.getExtractType().equals("Federated")) {
            int federatedQueryEnts = 0;
            SourceFederatedQueryConfigPojo endpoint = null;
            try {
                endpoint = source.getProcessingPipeline().get(0).federatedQuery;
            } catch (Exception e) {
            }
            if (null == endpoint) {
                rp.setResponse(
                        new ResponseObject("Test Source", false, "source error: no federated query specified"));
                return rp;
            }
            AdvancedQueryPojo testQuery = null;
            String errMessage = "no query specified";
            try {
                testQuery = AdvancedQueryPojo.fromApi(endpoint.testQueryJson, AdvancedQueryPojo.class);
            } catch (Exception e) {
                errMessage = e.getMessage();
            }
            if (null == testQuery) {
                rp.setResponse(new ResponseObject("Test Source", false,
                        "source error: need to specifiy a valid IKANOW query to test federated queries, error: "
                                + errMessage));
                return rp;
            }
            // OK if we're here then we can test the query
            SimpleFederatedQueryEngine testFederatedQuery = new SimpleFederatedQueryEngine();
            endpoint.parentSource = source;
            testFederatedQuery.addEndpoint(endpoint);
            ObjectId queryId = new ObjectId();
            String[] communityIdStrs = new String[source.getCommunityIds().size()];
            int i = 0;
            for (ObjectId commId : source.getCommunityIds()) {
                communityIdStrs[i] = commId.toString();
                i++;
            }
            testFederatedQuery.setTestMode(true);
            testFederatedQuery.preQueryActivities(queryId, testQuery, communityIdStrs);
            StatisticsPojo stats = new StatisticsPojo();
            stats.setSavedScores(0, 0);
            rp.setStats(stats);
            ArrayList<BasicDBObject> toAddTemp = new ArrayList<BasicDBObject>(1);
            testFederatedQuery.postQueryActivities(queryId, toAddTemp, rp);
            for (BasicDBObject docObj : toAddTemp) {
                DocumentPojo doc = DocumentPojo.fromDb(docObj, DocumentPojo.class);
                if (bReturnFullText) {
                    doc.setFullText(docObj.getString(DocumentPojo.fullText_));
                    doc.makeFullTextNonTransient();
                }
                if (null != doc.getEntities()) {
                    federatedQueryEnts += doc.getEntities().size();
                }

                //Metadata workaround:
                @SuppressWarnings("unchecked")
                LinkedHashMap<String, Object[]> meta = (LinkedHashMap<String, Object[]>) docObj
                        .get(DocumentPojo.metadata_);
                if (null != meta) {
                    Object metaJson = meta.get("json");
                    if (metaJson instanceof Object[]) { // (in this case ... non-cached, need to recopy in, I forget why)
                        doc.addToMetadata("json", (Object[]) metaJson);
                    }
                }
                toAdd.add(doc);
            }
            // (currently can't run harvest source federated query)
            if (0 == federatedQueryEnts) { // (more fed query exceptions)
                source.getHarvestStatus().setHarvest_message(
                        "Warning: no entities extracted, probably docConversionMap is wrong?");
            } else {
                source.getHarvestStatus().setHarvest_message(federatedQueryEnts + " entities extracted");
            }

        } //TESTED (END FEDERATED QUERY TEST MODE, WHICH IS A BIT DIFFERENT)
        else {
            harvester.harvestSource(source, toAdd, toUpdate, toRemove);
        }

        // (don't parrot the old message back - v confusing)
        if (oldMessage == source.getHarvestStatus().getHarvest_message()) { // (ptr ==)
            source.getHarvestStatus()
                    .setHarvest_message("(no documents extracted - likely a source or configuration error)");
        } //TESTED

        String message = null;
        if ((null != source.getHarvestStatus()) && (null != source.getHarvestStatus().getHarvest_message())) {
            message = source.getHarvestStatus().getHarvest_message();
        } else {
            message = "";
        }
        List<String> errMessagesFromSourceDeser = apiMap.getErrorMessages();
        if (null != errMessagesFromSourceDeser) {
            StringBuffer sbApiMapErr = new StringBuffer("Substitution errors:\n");
            for (String err : errMessagesFromSourceDeser) {
                sbApiMapErr.append(err).append("\n");
            }
            message = message + "\n" + sbApiMapErr.toString();
        } //TESTED (by hand)

        if ((null != source.getHarvestStatus())
                && (HarvestEnum.error == source.getHarvestStatus().getHarvest_status())) {
            rp.setResponse(new ResponseObject("Test Source", false, "source error: " + message));
            rp.setData(toAdd, new DocumentPojoApiMap());
        } else {
            if ((null == message) || message.isEmpty()) {
                message = "no messages from harvester";
            }
            rp.setResponse(new ResponseObject("Test Source", true,
                    "successfully returned " + toAdd.size() + " docs: " + message));
            try {
                // If grabbing full text
                // Also some logstash/custom specific logic - these aren't docs so just output the entire record
                boolean isLogstash = (null != source.getExtractType())
                        && source.getExtractType().equalsIgnoreCase("logstash");
                boolean isCustom = (null != source.getExtractType())
                        && source.getExtractType().equalsIgnoreCase("custom");
                List<BasicDBObject> records = null;
                if (bReturnFullText || isLogstash || isCustom) {
                    for (DocumentPojo doc : toAdd) {
                        if (isLogstash || isCustom) {
                            if (null == records) {
                                records = new ArrayList<BasicDBObject>(toAdd.size());
                            }
                            BasicDBObject dbo = (BasicDBObject) doc.getMetadata().get("record")[0];
                            Object test = dbo.get("_id");
                            if ((null != test) && (test instanceof ObjectId)) {
                                dbo.remove("_id"); // (unless it's a custom _id added from logstash then remove it)
                            }
                            records.add(dbo);
                        } //TESTED
                        else if (bReturnFullText) {
                            doc.makeFullTextNonTransient();
                        }
                    }
                } //TESTED
                if (null != records) {
                    rp.setData(records, (BasePojoApiMap<BasicDBObject>) null);
                } //TESTED
                else {
                    rp.setData(toAdd, new DocumentPojoApiMap());
                } //TESTED

                //Test deserialization:
                rp.toApi();
            } catch (Exception e) {
                //e.printStackTrace();
                StringBuffer sb = new StringBuffer();
                Globals.populateStackTrace(sb, e);
                rp.setData(
                        new BasicDBObject("error_message", "Error deserializing documents: " + sb.toString()),
                        null);
            }
        }
    } catch (Exception e) {
        // If an exception occurs log the error
        logger.error("Exception Message: " + e.getMessage(), e);
        rp.setResponse(new ResponseObject("Test Source", false, "Error testing source: " + e.getMessage()));
    } catch (Error e) {
        // If an exception occurs log the error
        logger.error("Exception Message: " + e.getMessage(), e);
        rp.setResponse(new ResponseObject("Test Source", false,
                "Configuration/Installation error: " + e.getMessage()));
    }
    return rp;
}

From source file:com.ikanow.infinit.e.api.knowledge.DocumentHandler.java

License:Open Source License

/**
 * Get information function that returns the user information in the form of a JSON String.
 * @param isAdmin //  w ww.  j a v  a 2s .  c  o  m
 * 
 * @param  key   the key definition of the user ( example email@email.com )
 * @return      a JSON string representation of the person information on success
 */
public ResponsePojo getInfo(String userIdStr, String sourceKey, String idStrOrUrl, boolean bReturnFullText,
        boolean returnRawData, boolean isAdmin) {
    ResponsePojo rp = new ResponsePojo();

    try {
        // Set up the query
        BasicDBObject query = new BasicDBObject();
        ObjectId id = null;
        if (null == sourceKey) {
            id = new ObjectId(idStrOrUrl);
            query.put(DocumentPojo._id_, id);
        } else {
            query.put(DocumentPojo.sourceKey_, sourceKey);
            query.put(DocumentPojo.url_, idStrOrUrl);
        }

        if (!isAdmin)
            query.put(DocumentPojo.communityId_,
                    new BasicDBObject(MongoDbManager.in_, SocialUtils.getUserCommunities(userIdStr)));
        // (use DBObject here because DocumentPojo is pretty big and this call could conceivably have perf implications)

        BasicDBObject fieldsQ = new BasicDBObject();
        if (!bReturnFullText) {
            fieldsQ.put(DocumentPojo.fullText_, 0); // (XML/JSON have fullText as part of pojo)
        }

        BasicDBObject dbo = (BasicDBObject) DbManager.getDocument().getMetadata().findOne(query, fieldsQ);

        if ((null == dbo) || ((null != dbo.get(DocumentPojo.url_))
                && dbo.getString(DocumentPojo.url_).startsWith("?DEL?"))) {
            if (null != id) { // this might be the update id...               
                query = new BasicDBObject(DocumentPojo.updateId_, id);
                dbo = (BasicDBObject) DbManager.getDocument().getMetadata().findOne(query, fieldsQ);
            }
        }
        //TESTED (update case, normal case, and intermediate case where both update and original still exist)

        if (null == dbo) {
            rp.setResponse(new ResponseObject("Doc Info", true, "Document not found"));
            return rp;
        }
        DocumentPojo dp = DocumentPojo.fromDb(dbo, DocumentPojo.class);
        if (bReturnFullText) {
            if (null == dp.getFullText()) { // (Some things like database records might have this stored already)
                byte[] storageArray = new byte[200000];
                DBCollection contentDB = DbManager.getDocument().getContent();
                BasicDBObject contentQ = new BasicDBObject(CompressedFullTextPojo.url_, dp.getUrl());
                contentQ.put(CompressedFullTextPojo.sourceKey_,
                        new BasicDBObject(MongoDbManager.in_, Arrays.asList(null, dp.getSourceKey())));
                BasicDBObject fields = new BasicDBObject(CompressedFullTextPojo.gzip_content_, 1);
                BasicDBObject dboContent = (BasicDBObject) contentDB.findOne(contentQ, fields);
                if (null != dboContent) {
                    byte[] compressedData = ((byte[]) dboContent.get(CompressedFullTextPojo.gzip_content_));
                    ByteArrayInputStream in = new ByteArrayInputStream(compressedData);
                    GZIPInputStream gzip = new GZIPInputStream(in);
                    int nRead = 0;
                    StringBuffer output = new StringBuffer();
                    while (nRead >= 0) {
                        nRead = gzip.read(storageArray, 0, 200000);
                        if (nRead > 0) {
                            String s = new String(storageArray, 0, nRead, "UTF-8");
                            output.append(s);
                        }
                    }
                    dp.setFullText(output.toString());
                    dp.makeFullTextNonTransient();
                }
            }
        } else if (!returnRawData) {
            dp.setFullText(null); // (obviously will normally contain full text anyway)
        } else // if ( returnRawData )
        {
            //check if the harvest type is file, return the file instead
            //if file is db return the json
            //get source
            SourcePojo source = getSourceFromKey(dp.getSourceKey());
            if (source.getExtractType().equals("File")) {
                //get file from harvester
                String fileURL = dp.getUrl();
                if (dp.getSourceUrl() != null)
                    fileURL = dp.getSourceUrl();
                byte[] bytes = FileHarvester.getFile(fileURL, source);
                if (bytes == null) {
                    // Try returning JSON instead
                    String json = ApiManager.mapToApi(dp, new DocumentPojoApiMap());
                    DocumentFileInterface dfp = new DocumentFileInterface();

                    dfp.bytes = json.getBytes();
                    dfp.mediaType = "application/json";

                    rp.setResponse(
                            new ResponseObject("Doc Info", true, "Document bytes returned successfully"));
                    rp.setData(dfp, null);
                    return rp;
                } else {
                    DocumentFileInterface dfp = new DocumentFileInterface();
                    dfp.bytes = bytes;
                    dfp.mediaType = getMediaType(fileURL);
                    rp.setResponse(
                            new ResponseObject("Doc Info", true, "Document bytes returned successfully"));
                    rp.setData(dfp, null);
                    return rp;
                }
            } else {
                String json = ApiManager.mapToApi(dp, new DocumentPojoApiMap());
                DocumentFileInterface dfp = new DocumentFileInterface();

                dfp.bytes = json.getBytes();
                dfp.mediaType = "application/json";

                rp.setResponse(new ResponseObject("Doc Info", true, "Document bytes returned successfully"));
                rp.setData(dfp, null);
                return rp;
            }
        }
        rp.setData(dp, new DocumentPojoApiMap());
        rp.setResponse(new ResponseObject("Doc Info", true, "Feed info returned successfully"));
    } //(end full text vs raw data)
    catch (Exception e) {
        // If an exception occurs log the error
        logger.error("Exception Message: " + e.getMessage(), e);
        rp.setResponse(new ResponseObject("Doc Info", false, "error returning feed: " + e.getMessage()));
    }
    // Return Json String representing the user
    return rp;
}

From source file:com.ikanow.infinit.e.api.knowledge.federated.SimpleFederatedQueryEngine.java

License:Open Source License

@Override
public void postQueryActivities(ObjectId queryId, List<BasicDBObject> docs, ResponsePojo response) {
    boolean grabbedScores = false;
    double aggregateSignif = 100.0;
    double queryRelevance = 100.0;
    double score = 100.0;

    if (null != _asyncRequestsPerQuery) {
        int added = 0;
        BasicDBList bsonArray = new BasicDBList();
        PeekingIterator<FederatedRequest> it = Iterators.peekingIterator(_asyncRequestsPerQuery.iterator());
        while (it.hasNext()) {
            // loop state:
            BasicDBObject[] docOrDocs = new BasicDBObject[1];
            docOrDocs[0] = null;//from  www  .  ja v a  2 s. c om

            FederatedRequest request = it.next();
            boolean isComplexSource = isComplexSource(request.endpointInfo.parentSource);
            if (null == request.cachedDoc) { // no cached doc, simple source processing (OR ANY COMPLEX CASE BY CONSTRUCTION)
                try {
                    if ((null == request.cachedResult) || isComplexSource) { // no cached api response, or complex         
                        if (null != request.importThread) {
                            // 1) wait for the thread to finish
                            if (null == request.endpointInfo.queryTimeout_secs) {
                                request.endpointInfo.queryTimeout_secs = 300;
                            }
                            for (int timer = 0; timer < request.endpointInfo.queryTimeout_secs; timer++) {
                                try {
                                    request.importThread.join(1000L);
                                    if (!request.importThread.isAlive()) {
                                        break;
                                    }
                                } //TESTED (by hand)
                                catch (Exception e) {
                                    //(carry on)
                                }
                            }
                            if (request.importThread.isAlive()) {
                                request.errorMessage = new RuntimeException("Script timed out");
                            } //TESTED (by hand)

                            // 2) Get the results
                            if (null != request.errorMessage) {
                                if (_testMode) {
                                    throw new RuntimeException(request.errorMessage);
                                }
                            } else if (isComplexSource) {
                                //DEBUG 
                                if (_DEBUG)
                                    _logger.debug("DEB: postQA0: " + request.complexSourceProcResults.size());

                                handleComplexDocCaching(request, _cacheMode, _scoreStats);

                                // Get a list of docs
                                docOrDocs = ((BasicDBList) DocumentPojo
                                        .listToDb(request.complexSourceProcResults, DocumentPojo.listType()))
                                                .toArray(new BasicDBObject[0]);

                                // (_API_ caching is exactly the same between cache and non-cache cases)
                                // (note that if null != complexSourceProcResults then follows that null != scriptResult)
                                String url = buildScriptUrl(request.mergeKey, request.queryIndex);

                                if (!(request.importThread instanceof FederatedSimpleHarvest) && _cacheMode) { // (don't cache python federated queries in test mode)
                                    // (simple harvest caching is done separately)
                                    this.cacheApiResponse(url, request.scriptResult, request.endpointInfo);
                                }
                            } //TESTED (by hand - single and multiple doc mode)               
                            else if (null == request.scriptResult) {
                                if (_testMode) {
                                    throw new RuntimeException("Script mode: no cached result found from: "
                                            + request.requestParameter);
                                }
                            } else {
                                // (_API_ caching is exactly the same between cache and non-cache cases)
                                String url = buildScriptUrl(request.mergeKey, request.queryIndex);
                                if (_cacheMode) { // (don't cache python federated queries in test mode)
                                    this.cacheApiResponse(url, request.scriptResult, request.endpointInfo);
                                }
                                bsonArray.add(request.scriptResult);
                            }
                        } // end script mode
                        else { // HTTP mode (also: must be simple source builder)
                            Response endpointResponse = request.responseFuture.get();
                            request.asyncClient.close();
                            request.asyncClient = null;

                            String jsonStr = endpointResponse.getResponseBody();
                            String url = endpointResponse.getUri().toURL().toString();

                            Object bsonUnknownType = com.mongodb.util.JSON.parse(jsonStr);
                            BasicDBObject bson = null;
                            if (bsonUnknownType instanceof BasicDBObject) {
                                bson = (BasicDBObject) bsonUnknownType;
                            } else if (bsonUnknownType instanceof BasicDBList) {
                                bson = new BasicDBObject(SimpleFederatedCache.array_, bsonUnknownType);
                            } else if (bsonUnknownType instanceof String) {
                                bson = new BasicDBObject(SimpleFederatedCache.value_, bsonUnknownType);
                            }

                            //DEBUG
                            if (_DEBUG)
                                _logger.debug("DEB: postQA1: " + url + ": " + jsonStr);

                            if (null != bson) {
                                MongoDbUtil.enforceTypeNamingPolicy(bson, 0);
                                this.cacheApiResponse(url, bson, request.endpointInfo);
                                bsonArray.add(bson);
                            }
                        } //(end script vs request method)
                    } //TESTED (3.1, 4.2)
                    else { // (just used cached value)
                        //DEBUG 
                        if (_DEBUG)
                            _logger.debug("DEB: postQA2: " + request.cachedResult.toString());

                        bsonArray.add(
                                (BasicDBObject) request.cachedResult.get(SimpleFederatedCache.cachedJson_));
                    } //TESTED (4.1, 4.3)
                } catch (Exception e) {
                    //DEBUG
                    if (null == request.subRequest) {
                        _logger.error("Error with script: " + e.getMessage());
                        if (_testMode) {
                            throw new RuntimeException("Error with script: " + e.getMessage(), e);
                        }
                    } else {
                        _logger.error("Error with " + request.subRequest.endPointUrl + ": " + e.getMessage());
                        if (_testMode) {
                            throw new RuntimeException(
                                    "Error with " + request.subRequest.endPointUrl + ": " + e.getMessage(), e);
                        }
                    }
                }

                if (null == docOrDocs[0]) {
                    // (this next bit of logic can only occur in simple source cases by construction, phew)
                    if (!it.hasNext() || (request.mergeKey != it.peek().mergeKey)) { // deliberate ptr arithmetic
                        String url = buildScriptUrl(request.mergeKey, request.queryIndex);

                        //DEBUG
                        if (_DEBUG)
                            _logger.debug("DEB: postQA3: " + url + ": " + bsonArray);

                        docOrDocs[0] = createDocFromJson(bsonArray, url, request, request.endpointInfo);
                    }
                }
            } // (end if no cached doc)
            else { // cached doc, bypass lots of processing because no merging and doc already built (simple source processing)
                docOrDocs[0] = request.cachedDoc;
            } //TESTED (by hand)

            if (null != docOrDocs[0])
                for (BasicDBObject doc : docOrDocs) {

                    // Cache the document unless already cached (or caching disabled)
                    if ((null == request.cachedDoc) && _cacheMode && !isComplexSource
                            && ((null == request.endpointInfo.cacheTime_days)
                                    || (request.endpointInfo.cacheTime_days >= 0))) {
                        simpleDocCache(request, doc);
                    } //TESTED (by hand, 3 cases: cached not expired, cached expired first time, cached expired multiple times)

                    if (!grabbedScores) {
                        if (!docs.isEmpty()) {
                            BasicDBObject topDoc = docs.get(0);
                            aggregateSignif = topDoc.getDouble(DocumentPojo.aggregateSignif_, aggregateSignif);
                            queryRelevance = topDoc.getDouble(DocumentPojo.queryRelevance_, queryRelevance);
                            score = topDoc.getDouble(DocumentPojo.score_, score);
                            grabbedScores = true;

                            // OK would also like to grab the original matching entity, if it exists
                            if (!isComplexSource) {
                                BasicDBList ents = (BasicDBList) topDoc.get(DocumentPojo.entities_);
                                if (null != ents) {
                                    for (Object entObj : ents) {
                                        BasicDBObject ent = (BasicDBObject) entObj;
                                        String entIndex = ent.getString(EntityPojo.index_, "");
                                        if (entIndex.equals(request.queryIndex)) {
                                            ents = (BasicDBList) doc.get(DocumentPojo.entities_);
                                            if (null != ents) {
                                                ents.add(ent);
                                            }
                                            break;
                                        }
                                    }
                                } //TESTED (by hand)
                            }
                        }
                    }
                    doc.put(DocumentPojo.aggregateSignif_, aggregateSignif);
                    doc.put(DocumentPojo.queryRelevance_, queryRelevance);
                    doc.put(DocumentPojo.score_, score);

                    // Swap id and updateId, everything's been cached now:
                    // Handle update ids vs normal ids:
                    ObjectId updateId = (ObjectId) doc.get(DocumentPojo.updateId_);
                    if (null != updateId) { // swap the 2...
                        doc.put(DocumentPojo.updateId_, doc.get(DocumentPojo._id_));
                        doc.put(DocumentPojo._id_, updateId);
                    } //TESTED (by hand)            

                    // If we're returning to a query then we'll adjust the doc format (some of the atomic fields become arrays)
                    if (!_testMode) {
                        convertDocToQueryFormat(doc, request.communityIdStrs);
                    } //TESTED (by hand)

                    docs.add(0, doc);
                    added++;
                    //(doc auto reset at top of loop)

                    //(end if built a doc from the last request/set of requests)
                } //TESTED (3.1)      

        } //(end loop over federated requests)

        if (null != response.getStats()) {
            response.getStats().found += added;
        } //TESTED (by hand)         
    }
}

From source file:com.ikanow.infinit.e.api.knowledge.federated.SimpleFederatedQueryEngine.java

License:Open Source License

private static BasicDBObject convertDocToQueryFormat(BasicDBObject doc, String[] communityIdStrs) {
    doc.put(DocumentPojo.sourceKey_, Arrays.asList(doc.get(DocumentPojo.sourceKey_))); // (don't need to remove #N/#NN - fed queries can't have them)
    doc.put(DocumentPojo.source_, Arrays.asList(doc.get(DocumentPojo.source_)));
    doc.put(DocumentPojo.mediaType_, Arrays.asList(doc.get(DocumentPojo.mediaType_)));
    doc.put(DocumentPojo.communityId_, communityIdStrs);

    return doc;/*  w ww. ja v  a 2s.c  o  m*/
}

From source file:com.ikanow.infinit.e.api.knowledge.federated.SimpleFederatedQueryEngine.java

License:Open Source License

private static boolean checkIfNeedToClearCache(BasicDBObject cachedVal, SourcePojo src) {
    BasicDBObject cachedJson = (BasicDBObject) cachedVal.get(SimpleFederatedCache.cachedJson_);
    if (null == cachedJson) {
        return true; // (corrupt cache)
    }//from w w w.  j av a  2s.  co  m
    boolean isComplexSrc = isComplexSource(src);
    if (isComplexSrc) { // check API response
        Date createdDate = (Date) cachedVal.get(SimpleFederatedCache.created_);
        if (null == createdDate) {
            return true; // (needs date for doc caching code so just remove from cache and re-create)
        }
    }
    if ((1 == cachedJson.size())
            && (cachedJson.get(SimpleFederatedCache.__infinite__value_) instanceof String)) {
        // ie if complex source, return false - no need to clear cache
        return !isComplexSrc;
    } else { // opposite
        return isComplexSrc;
    }
}

From source file:com.ikanow.infinit.e.api.knowledge.federated.SimpleFederatedQueryEngine.java

License:Open Source License

public void test_CheckIfDocAdded(String testName, List<BasicDBObject> docs) {

    // 1) Did a doc get added?

    if (2 != docs.size()) {
        System.out.println("*** " + testName + ": didn't add doc? " + docs.size());
        System.exit(-1);/*from  w w  w .j  a  va 2s  .c  o  m*/
    }

    // 2) Was it the right doc?

    BasicDBObject doc = docs.get(0);
    doc.remove(DocumentPojo._id_);
    doc.remove(DocumentPojo.created_);
    doc.remove(DocumentPojo.modified_);
    doc.remove(DocumentPojo.publishedDate_);

    String docToCheck = "{ \"displayUrl\" : \"http://test3_1\" , \"url\" : \"inf://federated/fakeendpoint.123/test3_1/testentityin\" , \"sourceKey\" : [ \"fakeendpoint.123\"] , \"source\" : [ \"fakeendpoint\"] , \"communityId\" : [ \"4c927585d591d31d7b37097a\"] , \"mediaType\" : [ \"Report\"] , \"metadata\" : { \"json\" : [ { \"test\" : { \"field\" : [ \"test3_1\" , \"test3_1\"] , \"field2\" : \"http://test3_1\"}}]} , \"title\" : \"fake endpoint: : test3_1: test3_1\" , \"entities\" : [ { \"disambiguated_name\" : \"test3_1\" , \"type\" : \"TestEntityOut\" , \"dimension\" : \"What\" , \"relevance\" : 1.0 , \"doccount\" : 1 , \"averageFreq\" : 1.0 , \"datasetSignificance\" : 10.0 , \"significance\" : 10.0 , \"frequency\" : 1.0 , \"index\" : \"test3_1/testentityout\" , \"queryCoverage\" : 100.0 , \"totalfrequency\" : 1.0}] , \"description\" : \"[\\n  {\\n    \\\"test\\\": {\\n      \\\"field\\\": [\\n        \\\"test3_1\\\",\\n        \\\"test3_1\\\"\\n      ],\\n      \\\"field2\\\": \\\"http://test3_1\\\"\\n    }\\n  }\\n]\" , \"aggregateSignif\" : 115.0 , \"queryRelevance\" : 105.0 , \"score\" : 110.0}";

    if (!docToCheck.equals(doc.toString())) {
        System.out.println(
                "*** " + testName + ": document incorrect:\n" + docToCheck + "\nVS\n" + doc.toString());
        System.exit(-1);
    }

    // 3) Did the doc get cached?

    DBCollection endpointCacheCollection = getCacheCollection();

    String hardwiredCacheId = "http://localhost:8186/test/test3_1/?param1=test3_1";
    BasicDBObject cachedVal = (BasicDBObject) endpointCacheCollection
            .findOne(new BasicDBObject(SimpleFederatedCache._id_, hardwiredCacheId));

    if (null == cachedVal) {
        System.out.println(
                "*** " + testName + ": no cache for: " + doc.get(DocumentPojo.url_) + " / " + hardwiredCacheId);
        System.exit(-1);
    } else {
        Date expiryDate = cachedVal.getDate(SimpleFederatedCache.expiryDate_, null);
        if ((null == expiryDate) || ((new Date().getTime() - expiryDate.getTime()) > 10 * 1000L)) {
            System.out.println("*** " + testName + ": expiry date for: " + doc.get(DocumentPojo.url_) + ": "
                    + expiryDate + " vs now= " + new Date());
            System.exit(-1);
        }
        BasicDBObject cachedJson = (BasicDBObject) cachedVal.get(SimpleFederatedCache.cachedJson_);
        Object docVal = null;
        try {
            Object[] docVals = (Object[]) ((BasicDBObject) doc.get(DocumentPojo.metadata_)).get("json");
            docVal = docVals[0];
        } catch (Exception e) {
            @SuppressWarnings("rawtypes")
            Collection docVals = (Collection) ((BasicDBObject) doc.get(DocumentPojo.metadata_)).get("json");
            docVal = docVals.iterator().next();

        }
        if ((null == cachedJson) || !cachedJson.equals(docVal)) {
            System.out.println("*** " + testName + ": cache: " + doc.get(DocumentPojo.url_)
                    + ": cached val incorrect:\n" + docVal + "\nVS\n" + cachedJson);
            System.exit(-1);
        }
    }
}