com.github.maasdi.mongo.wrapper.NoAuthMongoClientWrapper.java Source code

Java tutorial

Introduction

Here is the source code for com.github.maasdi.mongo.wrapper.NoAuthMongoClientWrapper.java

Source

/**
 * Copyright (C) 2014 Maas Dianto (maas.dianto@gmail.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.github.maasdi.mongo.wrapper;

import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.bson.types.BSONTimestamp;
import org.bson.types.Binary;
import org.bson.types.Code;
import org.bson.types.MaxKey;
import org.bson.types.MinKey;
import org.bson.types.ObjectId;
import org.bson.types.Symbol;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.logging.LogChannelInterface;
import org.pentaho.di.core.row.ValueMeta;
import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.core.variables.VariableSpace;
import org.pentaho.di.i18n.BaseMessages;

import com.mongodb.AggregationOutput;
import com.mongodb.BasicDBList;
import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.MongoClient;
import com.mongodb.MongoClientOptions;
import com.mongodb.ServerAddress;
import com.mongodb.TaggableReadPreference;
import com.mongodb.WriteConcern;
import com.mongodb.util.JSON;

import com.github.maasdi.di.trans.steps.mongodb.MongoDbMeta;
import com.github.maasdi.mongo.NamedReadPreference;
import com.github.maasdi.mongo.wrapper.collection.DefaultMongoDeleteCollectionWrapper;
import com.github.maasdi.mongo.wrapper.collection.MongoDeleteCollectionWrapper;
import com.github.maasdi.mongo.wrapper.field.MongoField;

public class NoAuthMongoClientWrapper implements MongoClientWrapper {
    private static Class<?> PKG = NoAuthMongoClientWrapper.class;
    public static final int MONGO_DEFAULT_PORT = 27017;

    public static final String LOCAL_DB = "local";
    public static final String REPL_SET_COLLECTION = "system.replset";
    public static final String REPL_SET_SETTINGS = "settings";
    public static final String REPL_SET_LAST_ERROR_MODES = "getLastErrorModes";
    public static final String REPL_SET_MEMBERS = "members";

    private final MongoClient mongo;
    private final LogChannelInterface log;

    /**
     * Create a connection to a Mongo server based on parameters supplied in the step meta data
     *
     * @param meta
     *          the step meta data
     * @param vars
     *          variables to use
     * @param cred
     *          a configured MongoCredential for authentication (or null for no authentication)
     * @param log
     *          for logging
     * @return a configured MongoClient object
     * @throws KettleException
     *           if a problem occurs
     */
    public NoAuthMongoClientWrapper(MongoDbMeta meta, VariableSpace vars, LogChannelInterface log)
            throws KettleException {
        this.log = log;
        mongo = initConnection(meta, vars, log);
    }

    public NoAuthMongoClientWrapper(MongoClient mongo, LogChannelInterface log) {
        this.mongo = mongo;
        this.log = log;
    }

    public MongoClient getMongo() {
        return mongo;
    }

    private MongoClient initConnection(MongoDbMeta meta, VariableSpace vars, LogChannelInterface log)
            throws KettleException {
        String hostsPorts = vars.environmentSubstitute(meta.getHostnames());
        String singlePort = vars.environmentSubstitute(meta.getPort());
        String connTimeout = meta.getConnectTimeout();
        String socketTimeout = meta.getSocketTimeout();
        String readPreference = meta.getReadPreference();
        String writeConcern = meta.getWriteConcern();
        String wTimeout = meta.getWTimeout();
        boolean journaled = meta.getJournal();
        List<String> tagSet = meta.getReadPrefTagSets();
        boolean useAllReplicaSetMembers = meta.getUseAllReplicaSetMembers();
        int singlePortI = -1;

        try {
            singlePortI = Integer.parseInt(singlePort);
        } catch (NumberFormatException n) {
            // don't complain
        }

        if (Const.isEmpty(hostsPorts)) {
            throw new KettleException(
                    BaseMessages.getString(PKG, "MongoNoAuthWrapper.Message.Error.EmptyHostsString")); //$NON-NLS-1$
        }

        List<ServerAddress> repSet = new ArrayList<ServerAddress>();

        // if (useAllReplicaSetMembers) {
        // repSet = getReplicaSetMembers(hostsPorts, singlePort, cred, vars, log);
        //
        // if (repSet.size() == 0) {
        // useAllReplicaSetMembers = false; // drop back and just configure using
        // // what we've been given
        // } else {
        // if (log != null) {
        // StringBuilder builder = new StringBuilder();
        // for (ServerAddress s : repSet) {
        // builder.append(s.toString()).append(" ");
        // }
        // log.logBasic(BaseMessages.getString(PKG,
        // "MongoUtils.Message.UsingTheFollowingReplicaSetMembers")
        // + " "
        // + builder.toString());
        // }
        // }
        // }

        // if (!useAllReplicaSetMembers) {
        String[] parts = hostsPorts.trim().split(","); //$NON-NLS-1$
        for (String part : parts) {
            // host:port?
            int port = singlePortI != -1 ? singlePortI : MONGO_DEFAULT_PORT;
            String[] hp = part.split(":"); //$NON-NLS-1$
            if (hp.length > 2) {
                throw new KettleException(
                        BaseMessages.getString(PKG, "MongoNoAuthWrapper.Message.Error.MalformedHost", part)); //$NON-NLS-1$
            }

            String host = hp[0];
            if (hp.length == 2) {
                // non-default port
                try {
                    port = Integer.parseInt(hp[1].trim());
                } catch (NumberFormatException n) {
                    throw new KettleException(BaseMessages.getString(PKG,
                            "MongoNoAuthWrapper.Message.Error.UnableToParsePortNumber", hp[1])); //$NON-NLS-1$
                }
            }

            try {
                ServerAddress s = new ServerAddress(host, port);
                repSet.add(s);
            } catch (UnknownHostException u) {
                throw new KettleException(u);
            }
        }
        // }

        MongoClientOptions.Builder mongoOptsBuilder = new MongoClientOptions.Builder();

        configureConnectionOptions(mongoOptsBuilder, connTimeout, socketTimeout, readPreference, writeConcern,
                wTimeout, journaled, tagSet, vars, log);

        MongoClientOptions opts = mongoOptsBuilder.build();
        return getClient(meta, vars, log, repSet, useAllReplicaSetMembers, opts);
    }

    protected MongoClient getClient(MongoDbMeta meta, VariableSpace vars, LogChannelInterface log,
            List<ServerAddress> repSet, boolean useAllReplicaSetMembers, MongoClientOptions opts)
            throws KettleException {
        try {
            // Mongo's java driver will discover all replica set or shard
            // members (Mongos) automatically when MongoClient is constructed
            // using a list of ServerAddresses. The javadocs state that MongoClient
            // should be constructed using a SingleServer address instance (rather
            // than a list) when connecting to a stand-alone host - this is why
            // we differentiate here between a list containing one ServerAddress
            // and a single ServerAddress instance via the useAllReplicaSetMembers
            // flag.
            return (repSet.size() > 1 || (useAllReplicaSetMembers && repSet.size() >= 1)
                    ? new MongoClient(repSet, opts)
                    : (repSet.size() == 1 ? new MongoClient(repSet.get(0), opts)
                            : new MongoClient(new ServerAddress("localhost"), opts))); //$NON-NLS-1$
        } catch (UnknownHostException u) {
            throw new KettleException(u);
        }
    }

    /**
     * Utility method to configure Mongo connection options
     *
     * @param optsBuilder
     *          an options builder
     * @param connTimeout
     *          the connection timeout to use (can be null)
     * @param socketTimeout
     *          the socket timeout to use (can be null)
     * @param readPreference
     *          the read preference to use (can be null)
     * @param writeConcern
     *          the writeConcern to use (can be null)
     * @param wTimeout
     *          the w timeout to use (can be null)
     * @param journaled
     *          whether to use journaled writes
     * @param tagSet
     *          the tag set to use in conjunction with the read preference (can be null)
     * @param vars
     *          variables to use
     * @param log
     *          for logging
     * @throws KettleException
     *           if a problem occurs
     */
    private void configureConnectionOptions(MongoClientOptions.Builder optsBuilder, String connTimeout,
            String socketTimeout, String readPreference, String writeConcern, String wTimeout, boolean journaled,
            List<String> tagSet, VariableSpace vars, LogChannelInterface log) throws KettleException {

        // connection timeout
        if (!Const.isEmpty(connTimeout)) {
            String connS = vars.environmentSubstitute(connTimeout);
            try {
                int cTimeout = Integer.parseInt(connS);
                if (cTimeout > 0) {
                    optsBuilder.connectTimeout(cTimeout);
                }
            } catch (NumberFormatException n) {
                throw new KettleException(n);
            }
        }

        // socket timeout
        if (!Const.isEmpty(socketTimeout)) {
            String sockS = vars.environmentSubstitute(socketTimeout);
            try {
                int sockTimeout = Integer.parseInt(sockS);
                if (sockTimeout > 0) {
                    optsBuilder.socketTimeout(sockTimeout);
                }
            } catch (NumberFormatException n) {
                throw new KettleException(n);
            }
        }

        if (log != null) {
            String rpLogSetting = NamedReadPreference.PRIMARY.getName();

            if (!Const.isEmpty(readPreference)) {
                rpLogSetting = readPreference;
            }
            log.logBasic(
                    BaseMessages.getString(PKG, "MongoNoAuthWrapper.Message.UsingReadPreference", rpLogSetting)); //$NON-NLS-1$
        }
        DBObject firstTagSet = null;
        DBObject[] remainingTagSets = new DBObject[0];
        if (tagSet != null && tagSet.size() > 0) {
            if (tagSet.size() > 1) {
                remainingTagSets = new DBObject[tagSet.size() - 1];
            }

            firstTagSet = (DBObject) JSON.parse(tagSet.get(0).trim());
            for (int i = 1; i < tagSet.size(); i++) {
                remainingTagSets[i - 1] = (DBObject) JSON.parse(tagSet.get(i).trim());
            }
            if (log != null && (!Const.isEmpty(readPreference)
                    && !readPreference.equalsIgnoreCase(NamedReadPreference.PRIMARY.getName()))) {
                StringBuilder builder = new StringBuilder();
                for (String s : tagSet) {
                    builder.append(s).append(" "); //$NON-NLS-1$
                }
                log.logBasic(BaseMessages.getString(PKG, "MongoNoAuthWrapper.Message.UsingReadPreferenceTagSets", //$NON-NLS-1$
                        builder.toString()));
            }
        } else {
            if (log != null) {
                log.logBasic(
                        BaseMessages.getString(PKG, "MongoNoAuthWrapper.Message.NoReadPreferenceTagSetsDefined")); //$NON-NLS-1$
            }
        }

        // read preference
        if (!Const.isEmpty(readPreference)) {

            String rp = vars.environmentSubstitute(readPreference);

            NamedReadPreference preference = NamedReadPreference.byName(rp);

            if ((firstTagSet != null) && (preference.getPreference() instanceof TaggableReadPreference)) {
                optsBuilder.readPreference(preference.getTaggableReadPreference(firstTagSet, remainingTagSets));
            } else {
                optsBuilder.readPreference(preference.getPreference());
            }

        }

        // write concern
        writeConcern = vars.environmentSubstitute(writeConcern);
        wTimeout = vars.environmentSubstitute(wTimeout);

        WriteConcern concern = null;

        if (Const.isEmpty(writeConcern) && Const.isEmpty(wTimeout) && !journaled) {
            // all defaults - timeout 0, journal = false, w = 1
            concern = new WriteConcern();
            concern.setWObject(new Integer(1));

            if (log != null) {
                log.logBasic(BaseMessages.getString(PKG,
                        "MongoNoAuthWrapper.Message.ConfiguringWithDefaultWriteConcern")); //$NON-NLS-1$
            }
        } else {
            int wt = 0;
            if (!Const.isEmpty(wTimeout)) {
                try {
                    wt = Integer.parseInt(wTimeout);
                } catch (NumberFormatException n) {
                    throw new KettleException(n);
                }
            }

            if (!Const.isEmpty(writeConcern)) {
                // try parsing as a number first
                try {
                    int wc = Integer.parseInt(writeConcern);
                    concern = new WriteConcern(wc, wt, false, journaled);
                } catch (NumberFormatException n) {
                    // assume its a valid string - e.g. "majority" or a custom
                    // getLastError label associated with a tag set
                    concern = new WriteConcern(writeConcern, wt, false, journaled);
                }
            } else {
                concern = new WriteConcern(1, wt, false, journaled);
            }

            if (log != null) {
                String lwc = "w = " + concern.getW() + ", wTimeout = " + concern.getWtimeout() + ", journaled = "
                        + concern.getJ();
                log.logBasic(
                        BaseMessages.getString(PKG, "MongoNoAuthWrapper.Message.ConfiguringWithWriteConcern", lwc));
            }
        }
        optsBuilder.writeConcern(concern);
    }

    /**
     * Retrieve all database names found in MongoDB as visible by the authenticated user.
     *
     * @throws KettleException
     */
    public List<String> getDatabaseNames() throws KettleException {
        try {
            return getMongo().getDatabaseNames();
        } catch (Exception e) {
            if (e instanceof KettleException) {
                throw (KettleException) e;
            } else {
                throw new KettleException(e);
            }
        }
    }

    protected DB getDb(String dbName) throws KettleException {
        try {
            return getMongo().getDB(dbName);
        } catch (Exception e) {
            if (e instanceof KettleException) {
                throw (KettleException) e;
            } else {
                throw new KettleException(e);
            }
        }
    }

    /**
     * Get the set of collections for a MongoDB database.
     *
     * @param meta
     *          Input meta with connection information
     * @param varSpace
     *          Variable space to substitute variables with
     * @param dB
     *          Name of database
     * @param username
     *          Username to request collections on behalf of
     * @param realPass
     *          Password of user
     * @return Set of collections in the database requested.
     * @throws KettleException
     *           If an error occurs.
     */
    public Set<String> getCollectionsNames(String dB) throws KettleException {
        try {
            return getDb(dB).getCollectionNames();
        } catch (Exception e) {
            if (e instanceof KettleException) {
                throw (KettleException) e;
            } else {
                throw new KettleException(e);
            }
        }
    }

    /**
     * Return a list of custom "lastErrorModes" (if any) defined in the replica set configuration object on the server.
     * These can be used as the "w" setting for the write concern in addition to the standard "w" values of <number> or
     * "majority".
     *
     * @return a list of the names of any custom "lastErrorModes"
     * @throws KettleException
     *           if a problem occurs
     */
    public List<String> getLastErrorModes() throws KettleException {
        List<String> customLastErrorModes = new ArrayList<String>();

        DB local = getDb(LOCAL_DB);
        if (local != null) {
            try {
                DBCollection replset = local.getCollection(REPL_SET_COLLECTION);
                if (replset != null) {
                    DBObject config = replset.findOne();

                    extractLastErrorModes(config, customLastErrorModes);
                }
            } catch (Exception e) {
                if (e instanceof KettleException) {
                    throw (KettleException) e;
                } else {
                    throw new KettleException(e);
                }
            }
        }

        return customLastErrorModes;
    }

    protected void extractLastErrorModes(DBObject config, List<String> customLastErrorModes) {
        if (config != null) {
            Object settings = config.get(REPL_SET_SETTINGS);

            if (settings != null) {
                Object getLastErrModes = ((DBObject) settings).get(REPL_SET_LAST_ERROR_MODES);

                if (getLastErrModes != null) {
                    for (String m : ((DBObject) getLastErrModes).keySet()) {
                        customLastErrorModes.add(m);
                    }
                }
            }
        }
    }

    public List<String> getIndexInfo(String dbName, String collection) throws KettleException {
        try {
            DB db = getDb(dbName);

            if (db == null) {
                throw new Exception(
                        BaseMessages.getString(PKG, "MongoNoAuthWrapper.ErrorMessage.NonExistentDB", dbName)); //$NON-NLS-1$
            }

            if (Const.isEmpty(collection)) {
                throw new Exception(
                        BaseMessages.getString(PKG, "MongoNoAuthWrapper.ErrorMessage.NoCollectionSpecified")); //$NON-NLS-1$
            }

            if (!db.collectionExists(collection)) {
                db.createCollection(collection, null);
            }

            DBCollection coll = db.getCollection(collection);
            if (coll == null) {
                throw new Exception(
                        BaseMessages.getString(PKG, "MongoNoAuthWrapper.ErrorMessage.UnableToGetInfoForCollection", //$NON-NLS-1$
                                collection));
            }

            List<DBObject> collInfo = coll.getIndexInfo();
            List<String> result = new ArrayList<String>();
            if (collInfo == null || collInfo.size() == 0) {
                throw new Exception(
                        BaseMessages.getString(PKG, "MongoNoAuthWrapper.ErrorMessage.UnableToGetInfoForCollection", //$NON-NLS-1$
                                collection));
            }

            for (DBObject index : collInfo) {
                result.add(index.toString());
            }

            return result;
        } catch (Exception e) {
            log.logError(BaseMessages.getString(PKG, "MongoNoAuthWrapper.ErrorMessage.GeneralError.Message") //$NON-NLS-1$
                    + ":\n\n" + e.getMessage(), e); //$NON-NLS-1$
            if (e instanceof KettleException) {
                throw (KettleException) e;
            } else {
                throw new KettleException(e);
            }
        }
    }

    public List<MongoField> discoverFields(String db, String collection, String query, String fields,
            boolean isPipeline, int docsToSample) throws KettleException {
        DBCursor cursor = null;
        try {
            int numDocsToSample = docsToSample;
            if (numDocsToSample < 1) {
                numDocsToSample = 100; // default
            }

            List<MongoField> discoveredFields = new ArrayList<MongoField>();
            Map<String, MongoField> fieldLookup = new HashMap<String, MongoField>();
            try {
                DB database = getDb(db);

                if (Const.isEmpty(collection)) {
                    throw new KettleException(
                            BaseMessages.getString(PKG, "MongoNoAuthWrapper.ErrorMessage.NoCollectionSpecified")); //$NON-NLS-1$
                }
                DBCollection dbcollection = database.getCollection(collection);

                Iterator<DBObject> pipeSample = null;

                if (isPipeline) {
                    pipeSample = setUpPipelineSample(query, numDocsToSample, dbcollection);
                } else {
                    if (Const.isEmpty(query) && Const.isEmpty(fields)) {
                        cursor = dbcollection.find().limit(numDocsToSample);
                    } else {
                        DBObject dbObject = (DBObject) JSON.parse(Const.isEmpty(query) ? "{}" //$NON-NLS-1$
                                : query);
                        DBObject dbObject2 = (DBObject) JSON.parse(fields);
                        cursor = dbcollection.find(dbObject, dbObject2).limit(numDocsToSample);
                    }
                }

                int actualCount = 0;
                while (cursor != null ? cursor.hasNext() : pipeSample.hasNext()) {
                    actualCount++;
                    DBObject nextDoc = (cursor != null ? cursor.next() : pipeSample.next());
                    docToFields(nextDoc, fieldLookup);
                }

                postProcessPaths(fieldLookup, discoveredFields, actualCount);

                return discoveredFields;
            } catch (Exception e) {
                throw new KettleException(e);
            } finally {
                if (cursor != null) {
                    cursor.close();
                }
            }
        } catch (Exception ex) {
            if (ex instanceof KettleException) {
                throw (KettleException) ex;
            } else {
                throw new KettleException(
                        BaseMessages.getString(PKG, "MongoNoAuthWrapper.ErrorMessage.UnableToDiscoverFields"), ex); //$NON-NLS-1$
            }
        }
    }

    private static Iterator<DBObject> setUpPipelineSample(String query, int numDocsToSample,
            DBCollection collection) throws KettleException {

        query = query + ", {$limit : " + numDocsToSample + "}"; //$NON-NLS-1$ //$NON-NLS-2$
        List<DBObject> samplePipe = jsonPipelineToDBObjectList(query);

        DBObject first = samplePipe.get(0);
        DBObject[] remainder = new DBObject[samplePipe.size() - 1];
        for (int i = 1; i < samplePipe.size(); i++) {
            remainder[i - 1] = samplePipe.get(i);
        }

        AggregationOutput result = collection.aggregate(first, remainder);

        return result.results().iterator();
    }

    public static List<DBObject> jsonPipelineToDBObjectList(String jsonPipeline) throws KettleException {
        List<DBObject> pipeline = new ArrayList<DBObject>();
        StringBuilder b = new StringBuilder(jsonPipeline.trim());

        // extract the parts of the pipeline
        int bracketCount = -1;
        List<String> parts = new ArrayList<String>();
        int i = 0;
        while (i < b.length()) {
            if (b.charAt(i) == '{') {
                if (bracketCount == -1) {
                    // trim anything off before this point
                    b.delete(0, i);
                    bracketCount = 0;
                    i = 0;
                }
                bracketCount++;
            }
            if (b.charAt(i) == '}') {
                bracketCount--;
            }
            if (bracketCount == 0) {
                String part = b.substring(0, i + 1);
                parts.add(part);
                bracketCount = -1;

                if (i == b.length() - 1) {
                    break;
                }
                b.delete(0, i + 1);
                i = 0;
            }

            i++;
        }

        for (String p : parts) {
            if (!Const.isEmpty(p)) {
                DBObject o = (DBObject) JSON.parse(p);
                pipeline.add(o);
            }
        }

        if (pipeline.size() == 0) {
            throw new KettleException(
                    BaseMessages.getString(PKG, "MongoNoAuthWrapper.ErrorMessage.UnableToParsePipelineOperators")); //$NON-NLS-1$
        }

        return pipeline;
    }

    protected static void docToFields(DBObject doc, Map<String, MongoField> lookup) {
        String root = "$"; //$NON-NLS-1$
        String name = "$"; //$NON-NLS-1$

        if (doc instanceof BasicDBObject) {
            processRecord((BasicDBObject) doc, root, name, lookup);
        } else if (doc instanceof BasicDBList) {
            processList((BasicDBList) doc, root, name, lookup);
        }
    }

    private static void processRecord(BasicDBObject rec, String path, String name, Map<String, MongoField> lookup) {
        for (String key : rec.keySet()) {
            Object fieldValue = rec.get(key);

            if (fieldValue instanceof BasicDBObject) {
                processRecord((BasicDBObject) fieldValue, path + "." + key, name + "." //$NON-NLS-1$ //$NON-NLS-2$
                        + key, lookup);
            } else if (fieldValue instanceof BasicDBList) {
                processList((BasicDBList) fieldValue, path + "." + key, name + "." //$NON-NLS-1$ //$NON-NLS-2$
                        + key, lookup);
            } else {
                // some sort of primitive
                String finalPath = path + "." + key; //$NON-NLS-1$
                String finalName = name + "." + key; //$NON-NLS-1$
                if (!lookup.containsKey(finalPath)) {
                    MongoField newField = new MongoField();
                    int kettleType = mongoToKettleType(fieldValue);
                    // Following suit of mongoToKettleType by interpreting null as String type
                    newField.m_mongoType = String.class;
                    if (fieldValue != null) {
                        newField.m_mongoType = fieldValue.getClass();
                    }
                    newField.m_fieldName = finalName;
                    newField.m_fieldPath = finalPath;
                    newField.m_kettleType = ValueMeta.getTypeDesc(kettleType);
                    newField.m_percentageOfSample = 1;

                    lookup.put(finalPath, newField);
                } else {
                    // update max indexes in array parts of name
                    MongoField m = lookup.get(finalPath);
                    Class<?> fieldClass = String.class;
                    if (fieldValue != null) {
                        fieldClass = fieldValue.getClass();
                    }
                    if (!m.m_mongoType.isAssignableFrom(fieldClass)) {
                        m.m_disparateTypes = true;
                    }
                    m.m_percentageOfSample++;
                    updateMaxArrayIndexes(m, finalName);
                }
            }
        }
    }

    protected static int mongoToKettleType(Object fieldValue) {
        if (fieldValue == null) {
            return ValueMetaInterface.TYPE_STRING;
        }

        if (fieldValue instanceof Symbol || fieldValue instanceof String || fieldValue instanceof Code
                || fieldValue instanceof ObjectId || fieldValue instanceof MinKey || fieldValue instanceof MaxKey) {
            return ValueMetaInterface.TYPE_STRING;
        } else if (fieldValue instanceof Date) {
            return ValueMetaInterface.TYPE_DATE;
        } else if (fieldValue instanceof Number) {
            // try to parse as an Integer
            try {
                Integer.parseInt(fieldValue.toString());
                return ValueMetaInterface.TYPE_INTEGER;
            } catch (NumberFormatException e) {
                return ValueMetaInterface.TYPE_NUMBER;
            }
        } else if (fieldValue instanceof Binary) {
            return ValueMetaInterface.TYPE_BINARY;
        } else if (fieldValue instanceof BSONTimestamp) {
            return ValueMetaInterface.TYPE_INTEGER;
        }

        return ValueMetaInterface.TYPE_STRING;
    }

    private static void processList(BasicDBList list, String path, String name, Map<String, MongoField> lookup) {

        if (list.size() == 0) {
            return; // can't infer anything about an empty list
        }

        String nonPrimitivePath = path + "[-]"; //$NON-NLS-1$
        String primitivePath = path;

        for (int i = 0; i < list.size(); i++) {
            Object element = list.get(i);

            if (element instanceof BasicDBObject) {
                processRecord((BasicDBObject) element, nonPrimitivePath, name + "[" + i //$NON-NLS-1$
                        + ":" + i + "]", lookup); //$NON-NLS-1$ //$NON-NLS-2$
            } else if (element instanceof BasicDBList) {
                processList((BasicDBList) element, nonPrimitivePath, name + "[" + i //$NON-NLS-1$
                        + ":" + i + "]", lookup); //$NON-NLS-1$ //$NON-NLS-2$
            } else {
                // some sort of primitive
                String finalPath = primitivePath + "[" + i + "]"; //$NON-NLS-1$ //$NON-NLS-2$
                String finalName = name + "[" + i + "]"; //$NON-NLS-1$ //$NON-NLS-2$
                if (!lookup.containsKey(finalPath)) {
                    MongoField newField = new MongoField();
                    int kettleType = mongoToKettleType(element);
                    // Following suit of mongoToKettleType by interpreting null as String type
                    newField.m_mongoType = String.class;
                    if (element != null) {
                        newField.m_mongoType = element.getClass();
                    }
                    newField.m_fieldName = finalPath;
                    newField.m_fieldPath = finalName;
                    newField.m_kettleType = ValueMeta.getTypeDesc(kettleType);
                    newField.m_percentageOfSample = 1;

                    lookup.put(finalPath, newField);
                } else {
                    // update max indexes in array parts of name
                    MongoField m = lookup.get(finalPath);
                    Class<?> elementClass = String.class;
                    if (element != null) {
                        elementClass = element.getClass();
                    }
                    if (!m.m_mongoType.isAssignableFrom(elementClass)) {
                        m.m_disparateTypes = true;
                    }
                    m.m_percentageOfSample++;
                    updateMaxArrayIndexes(m, finalName);
                }
            }
        }
    }

    protected static void postProcessPaths(Map<String, MongoField> fieldLookup, List<MongoField> discoveredFields,
            int numDocsProcessed) {
        for (String key : fieldLookup.keySet()) {
            MongoField m = fieldLookup.get(key);
            m.m_occurenceFraction = "" + m.m_percentageOfSample + "/" //$NON-NLS-1$ //$NON-NLS-2$
                    + numDocsProcessed;
            setMinArrayIndexes(m);

            // set field names to terminal part and copy any min:max array index
            // info
            if (m.m_fieldName.contains("[") && m.m_fieldName.contains(":")) { //$NON-NLS-1$ //$NON-NLS-2$
                m.m_arrayIndexInfo = m.m_fieldName;
            }
            if (m.m_fieldName.indexOf('.') >= 0) {
                m.m_fieldName = m.m_fieldName.substring(m.m_fieldName.lastIndexOf('.') + 1, m.m_fieldName.length());
            }

            if (m.m_disparateTypes) {
                // force type to string if we've seen this path more than once
                // with incompatible types
                m.m_kettleType = ValueMeta.getTypeDesc(ValueMeta.TYPE_STRING);
            }
            discoveredFields.add(m);
        }

        // check for name clashes
        Map<String, Integer> tempM = new HashMap<String, Integer>();
        for (MongoField m : discoveredFields) {
            if (tempM.get(m.m_fieldName) != null) {
                Integer toUse = tempM.get(m.m_fieldName);
                String key = m.m_fieldName;
                m.m_fieldName = key + "_" + toUse; //$NON-NLS-1$
                toUse = new Integer(toUse.intValue() + 1);
                tempM.put(key, toUse);
            } else {
                tempM.put(m.m_fieldName, 1);
            }
        }
    }

    protected static void setMinArrayIndexes(MongoField m) {
        // set the actual index for each array in the path to the
        // corresponding minimum index
        // recorded in the name

        if (m.m_fieldName.indexOf('[') < 0) {
            return;
        }

        String temp = m.m_fieldPath;
        String tempComp = m.m_fieldName;
        StringBuffer updated = new StringBuffer();

        while (temp.indexOf('[') >= 0) {
            String firstPart = temp.substring(0, temp.indexOf('['));
            String innerPart = temp.substring(temp.indexOf('[') + 1, temp.indexOf(']'));

            if (!innerPart.equals("-")) { //$NON-NLS-1$
                // terminal primitive specific index
                updated.append(temp); // finished
                temp = ""; //$NON-NLS-1$
                break;
            } else {
                updated.append(firstPart);

                String innerComp = tempComp.substring(tempComp.indexOf('[') + 1, tempComp.indexOf(']'));

                if (temp.indexOf(']') < temp.length() - 1) {
                    temp = temp.substring(temp.indexOf(']') + 1, temp.length());
                    tempComp = tempComp.substring(tempComp.indexOf(']') + 1, tempComp.length());
                } else {
                    temp = ""; //$NON-NLS-1$
                }

                String[] compParts = innerComp.split(":"); //$NON-NLS-1$
                String replace = "[" + compParts[0] + "]"; //$NON-NLS-1$ //$NON-NLS-2$
                updated.append(replace);

            }
        }

        if (temp.length() > 0) {
            // append remaining part
            updated.append(temp);
        }

        m.m_fieldPath = updated.toString();
    }

    protected static void updateMaxArrayIndexes(MongoField m, String update) {
        // just look at the second (i.e. max index value) in the array parts
        // of update
        if (m.m_fieldName.indexOf('[') < 0) {
            return;
        }

        if (m.m_fieldName.split("\\[").length != update.split("\\[").length) { //$NON-NLS-1$ //$NON-NLS-2$
            throw new IllegalArgumentException("Field path and update path do not seem to contain " //$NON-NLS-1$
                    + "the same number of array parts!"); //$NON-NLS-1$
        }

        String temp = m.m_fieldName;
        String tempComp = update;
        StringBuffer updated = new StringBuffer();

        while (temp.indexOf('[') >= 0) {
            String firstPart = temp.substring(0, temp.indexOf('['));
            String innerPart = temp.substring(temp.indexOf('[') + 1, temp.indexOf(']'));

            if (innerPart.indexOf(':') < 0) {
                // terminal primitive specific index
                updated.append(temp); // finished
                temp = ""; //$NON-NLS-1$
                break;
            } else {
                updated.append(firstPart);

                String innerComp = tempComp.substring(tempComp.indexOf('[') + 1, tempComp.indexOf(']'));

                if (temp.indexOf(']') < temp.length() - 1) {
                    temp = temp.substring(temp.indexOf(']') + 1, temp.length());
                    tempComp = tempComp.substring(tempComp.indexOf(']') + 1, tempComp.length());
                } else {
                    temp = ""; //$NON-NLS-1$
                }

                String[] origParts = innerPart.split(":"); //$NON-NLS-1$
                String[] compParts = innerComp.split(":"); //$NON-NLS-1$
                int origMax = Integer.parseInt(origParts[1]);
                int compMax = Integer.parseInt(compParts[1]);

                if (compMax > origMax) {
                    // updated the max index seen for this path
                    String newRange = "[" + origParts[0] + ":" + compMax + "]"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
                    updated.append(newRange);
                } else {
                    String oldRange = "[" + innerPart + "]"; //$NON-NLS-1$ //$NON-NLS-2$
                    updated.append(oldRange);
                }
            }
        }

        if (temp.length() > 0) {
            // append remaining part
            updated.append(temp);
        }

        m.m_fieldName = updated.toString();
    }

    @Override
    public List<String> getAllTags() throws KettleException {
        return setupAllTags(getRepSetMemberRecords());
    }

    private BasicDBList getRepSetMemberRecords() throws KettleException {
        BasicDBList setMembers = null;
        try {
            DB local = getDb(LOCAL_DB);
            if (local != null) {

                DBCollection replset = local.getCollection(REPL_SET_COLLECTION);
                if (replset != null) {
                    DBObject config = replset.findOne();

                    if (config != null) {
                        Object members = config.get(REPL_SET_MEMBERS);

                        if (members instanceof BasicDBList) {
                            if (((BasicDBList) members).size() == 0) {
                                // log that there are no replica set members defined
                                if (log != null) {
                                    log.logBasic(BaseMessages.getString(PKG,
                                            "MongoNoAuthWrapper.Message.Warning.NoReplicaSetMembersDefined")); //$NON-NLS-1$
                                }
                            } else {
                                setMembers = (BasicDBList) members;
                            }

                        } else {
                            // log that there are no replica set members defined
                            if (log != null) {
                                log.logBasic(BaseMessages.getString(PKG,
                                        "MongoNoAuthWrapper.Message.Warning.NoReplicaSetMembersDefined")); //$NON-NLS-1$
                            }
                        }
                    } else {
                        // log that there are no replica set members defined
                        if (log != null) {
                            log.logBasic(BaseMessages.getString(PKG,
                                    "MongoNoAuthWrapper.Message.Warning.NoReplicaSetMembersDefined")); //$NON-NLS-1$
                        }
                    }
                } else {
                    // log that the replica set collection is not available
                    if (log != null) {
                        log.logBasic(BaseMessages.getString(PKG,
                                "MongoNoAuthWrapper.Message.Warning.ReplicaSetCollectionUnavailable")); //$NON-NLS-1$
                    }
                }
            } else {
                // log that the local database is not available!!
                if (log != null) {
                    log.logBasic(
                            BaseMessages.getString(PKG, "MongoNoAuthWrapper.Message.Warning.LocalDBNotAvailable")); //$NON-NLS-1$
                }
            }
        } catch (Exception ex) {
            throw new KettleException(ex);
        } finally {
            if (getMongo() != null) {
                getMongo().close();
            }
        }

        return setMembers;
    }

    protected List<String> setupAllTags(BasicDBList members) {
        HashSet<String> tempTags = new HashSet<String>();

        if (members != null && members.size() > 0) {
            for (int i = 0; i < members.size(); i++) {
                Object m = members.get(i);

                if (m != null) {
                    DBObject tags = (DBObject) ((DBObject) m).get("tags"); //$NON-NLS-1$
                    if (tags == null) {
                        continue;
                    }

                    for (String tagName : tags.keySet()) {
                        String tagVal = tags.get(tagName).toString();
                        String combined = quote(tagName) + " : " + quote(tagVal); //$NON-NLS-1$
                        tempTags.add(combined);
                    }
                }
            }
        }

        return new ArrayList<String>(tempTags);
    }

    protected static String quote(String string) {
        if (string.indexOf('"') >= 0) {

            if (string.indexOf('"') >= 0) {
                string = string.replace("\"", "\\\""); //$NON-NLS-1$ //$NON-NLS-2$
            }
        }

        string = ("\"" + string + "\""); //$NON-NLS-1$ //$NON-NLS-2$

        return string;
    }

    /**
     * Return a list of replica set members whos tags satisfy the supplied list of tag set. It is assumed that members
     * satisfy according to an OR relationship = i.e. a member satisfies if it satisfies at least one of the tag sets in
     * the supplied list.
     *
     * @param tagSets
     *          the list of tag sets to match against
     * @return a list of replica set members who's tags satisfy the supplied list of tag sets
     * @throws KettleException
     *           if a problem occurs
     */
    public List<String> getReplicaSetMembersThatSatisfyTagSets(List<DBObject> tagSets) throws KettleException {
        try {
            List<String> result = new ArrayList<String>();
            for (DBObject object : checkForReplicaSetMembersThatSatisfyTagSets(tagSets, getRepSetMemberRecords())) {
                result.add(object.toString());
            }
            return result;
        } catch (Exception ex) {
            if (ex instanceof KettleException) {
                throw (KettleException) ex;
            } else {
                throw new KettleException(
                        BaseMessages.getString(PKG, "MongoNoAuthWrapper.ErrorMessage.UnableToGetReplicaSetMembers"), //$NON-NLS-1$
                        ex);
            }
        }
    }

    protected List<DBObject> checkForReplicaSetMembersThatSatisfyTagSets(List<DBObject> tagSets,
            BasicDBList members) {
        List<DBObject> satisfy = new ArrayList<DBObject>();
        if (members != null && members.size() > 0) {
            for (int i = 0; i < members.size(); i++) {
                Object m = members.get(i);

                if (m != null) {
                    DBObject tags = (DBObject) ((DBObject) m).get("tags"); //$NON-NLS-1$
                    if (tags == null) {
                        continue;
                    }

                    for (int j = 0; j < tagSets.size(); j++) {
                        boolean match = true;
                        DBObject toMatch = tagSets.get(j);

                        for (String tagName : toMatch.keySet()) {
                            String tagValue = toMatch.get(tagName).toString();

                            // does replica set member m's tags contain this tag?
                            Object matchVal = tags.get(tagName);

                            if (matchVal == null) {
                                match = false; // doesn't match this particular tag set
                                // no need to check any other keys in toMatch
                                break;
                            }

                            if (!matchVal.toString().equals(tagValue)) {
                                // rep set member m's tags has this tag, but it's value does not
                                // match
                                match = false;

                                // no need to check any other keys in toMatch
                                break;
                            }
                        }

                        if (match) {
                            // all tag/values present and match - add this member (only if its
                            // not already there)
                            if (!satisfy.contains(m)) {
                                satisfy.add((DBObject) m);
                            }
                        }
                    }
                }
            }
        }

        return satisfy;
    }

    @Override
    public MongoDeleteCollectionWrapper getCollection(String db, String name) throws KettleException {
        return wrap(getDb(db).getCollection(name));
    }

    @Override
    public MongoDeleteCollectionWrapper createCollection(String db, String name) throws KettleException {
        return wrap(getDb(db).createCollection(name, null));
    }

    protected MongoDeleteCollectionWrapper wrap(DBCollection collection) {
        return new DefaultMongoDeleteCollectionWrapper(collection);
    }

    @Override
    public void dispose() {
        getMongo().close();
    }
}