fr.gouv.vitam.mdbes.MainIngestESFromFile.java Source code

Java tutorial

Introduction

Here is the source code for fr.gouv.vitam.mdbes.MainIngestESFromFile.java

Source

/**
 * This file is part of POC MongoDB ElasticSearch Project.
 *
 * Copyright 2009, Frederic Bregier, and individual contributors by the @author tags. See the
 * COPYRIGHT.txt in the distribution for a full listing of individual contributors.
 *
 * All POC MongoDB ElasticSearch Project is free software: you can redistribute it and/or modify it under the terms of
 * the GNU General Public License as published by the Free Software Foundation, either versionRank 3
 * of the License, or (at your option) any later versionRank.
 *
 * POC MongoDB ElasticSearch is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
 * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
 * Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along with POC MongoDB ElasticSearch . If not, see
 * <http://www.gnu.org/licenses/>.
 */
package fr.gouv.vitam.mdbes;

import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.HashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;

import org.apache.log4j.PropertyConfigurator;
import org.bson.BSONObject;

import com.mongodb.MongoClient;
import com.mongodb.MongoClientOptions;
import com.mongodb.ReadPreference;
import com.mongodb.util.JSON;

import fr.gouv.vitam.query.GlobalDatas;

/**
 * Load ingest file into ElasticSearch
 *
 * @author "Frederic Bregier"
 *
 */
@SuppressWarnings("javadoc")
public class MainIngestESFromFile {
    private static AtomicLong loadt = new AtomicLong(0);

    private static MongoClient mongoClient = null;

    public static String model;
    public static String host = "localhost";
    public static String esbase = "vitam";
    public static String unicast = "mdb002, mdb003, mdb004";

    /**
     * @param args
     */
    public static void main(final String[] args) throws Exception {
        if (args.length < 6) {
            System.err.println("need: logfile host escluster unicast indextype files");
            return;
        }
        final String log4j = args[0];
        PropertyConfigurator.configure(log4j);
        final String networkHost = "192.168.56.102";
        GlobalDatas.localNetworkAddress = networkHost;
        // connect to the local database server
        if (args.length > 1) {
            host = args[1];
        }
        if (args.length > 2) {
            esbase = args[2];
        }
        if (args.length > 3) {
            unicast = args[3];
        }
        if (args.length > 4) {
            model = args[4];
        }
        MongoDbAccess dbvitam = null;
        try {
            final MongoClientOptions options = new MongoClientOptions.Builder().connectionsPerHost(4).build();
            mongoClient = new MongoClient(host, options);
            mongoClient.setReadPreference(ReadPreference.primaryPreferred());
            dbvitam = new MongoDbAccess(mongoClient, "VitamLinks", esbase, unicast, false);
            dbvitam.updateEsIndex(model);
            MainIngestESFromFile.loadt = new AtomicLong(0);
            MainIngestFile.cptMaip.set(0);
            for (int i = 5; i < args.length - 1; i++) {
                System.out.println("Load " + args[i]);
                runOnce(dbvitam, args[i], true);
            }
            int i = args.length - 1;
            System.out.println("Load " + args[i]);
            runOnce(dbvitam, args[i], false);
        } catch (final Exception e) {
            System.err.println("ERROR: " + e.getMessage());
            e.printStackTrace();

        } finally {
            // release resources
            final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(2);
            final ToClean toclean = new ToClean(dbvitam);
            scheduler.schedule(toclean, 1, TimeUnit.MILLISECONDS);
            final ToShutdown toShutdown = new ToShutdown();
            scheduler.schedule(toShutdown, 5000, TimeUnit.MILLISECONDS);
            scheduler.awaitTermination(7000, TimeUnit.MILLISECONDS);
            System.exit(0);
        }

    }

    private static final class ToClean implements Runnable {
        MongoDbAccess dbvitam;

        public ToClean(final MongoDbAccess dbvitam) {
            this.dbvitam = dbvitam;
        }

        @Override
        public void run() {
            dbvitam.close();
            mongoClient.close();
        }

    }

    private static final class ToShutdown implements Runnable {

        @Override
        public void run() {
            System.exit(0);
        }

    }

    private static final void runOnce(final MongoDbAccess dbvitam, final String file, boolean insert)
            throws InterruptedException, InstantiationException, IllegalAccessException, IOException {
        System.out.println("Load starting... ");

        final long date11 = System.currentTimeMillis();

        final HashMap<String, String> esIndex = new HashMap<>();
        final FileInputStream fstream = new FileInputStream(file);
        final DataInputStream in = new DataInputStream(fstream);
        final BufferedReader br = new BufferedReader(new InputStreamReader(in));
        String strLine;
        // Read File Line By Line
        while ((strLine = br.readLine()) != null) {
            final BSONObject bson = (BSONObject) JSON.parse(strLine);
            if (insert) {
                final int nbEs = ElasticSearchAccess.addEsIndex(dbvitam, model, esIndex, bson);
                MainIngestFile.cptMaip.addAndGet(nbEs);
            } else {
                ElasticSearchAccess.addEsIndex(dbvitam, model, bson);
                MainIngestFile.cptMaip.incrementAndGet();
            }
        }
        // Close the input stream
        br.close();
        in.close();
        fstream.close();
        if (!esIndex.isEmpty()) {
            MainIngestFile.cptMaip.addAndGet(esIndex.size());
            System.out.println("Last bulk ES");
            dbvitam.addEsEntryIndex(true, esIndex, model);
            esIndex.clear();
        }

        final long date12 = System.currentTimeMillis();
        MainIngestESFromFile.loadt.addAndGet(date12 - date11);

        System.out.println("Load ended");
        /*
         * System.out.println("All elements\n================================================================");
         * DbVitam.printStructure(dbvitam);
         */
        final long nbBigM = dbvitam.getDaipSize();
        final long nbBigD = dbvitam.getPaipSize();
        System.out.println("\n Big Test (" + MainIngestFile.nbThread + " Threads chacune " + MainIngestFile.nb
                + " itrations de load, nb MAIP: " + MainIngestFile.cptMaip.get() + ") with MAIP: " + nbBigM
                + " DATA: " + nbBigD + " => Load:"
                + (MainIngestESFromFile.loadt.get()) / ((float) MainIngestFile.cptMaip.get()));

        System.out.println("\nThread;nbLoad;nbTotal;Load");
        System.out.println(MainIngestFile.nbThread + ";" + MainIngestFile.cptMaip.get() + ";" + nbBigM + ";"
                + (MainIngestESFromFile.loadt.get()) / ((float) MainIngestFile.cptMaip.get()));
    }

}