fr.gouv.vitam.mdbes.MainPrepareIngestFile.java Source code

Java tutorial

Introduction

Here is the source code for fr.gouv.vitam.mdbes.MainPrepareIngestFile.java

Source

/**
 * This file is part of POC MongoDB ElasticSearch Project.
 *
 * Copyright 2009, Frederic Bregier, and individual contributors by the @author tags. See the
 * COPYRIGHT.txt in the distribution for a full listing of individual contributors.
 *
 * All POC MongoDB ElasticSearch Project is free software: you can redistribute it and/or modify it under the terms of
 * the GNU General Public License as published by the Free Software Foundation, either versionRank 3
 * of the License, or (at your option) any later versionRank.
 *
 * POC MongoDB ElasticSearch is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
 * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
 * Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along with POC MongoDB ElasticSearch . If not, see
 * <http://www.gnu.org/licenses/>.
 */
package fr.gouv.vitam.mdbes;

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;

import org.apache.log4j.PropertyConfigurator;

import com.mongodb.DBCursor;
import com.mongodb.MongoClient;
import com.mongodb.MongoClientOptions;
import com.mongodb.ReadPreference;

import fr.gouv.vitam.query.GlobalDatas;
import fr.gouv.vitam.query.exception.InvalidExecOperationException;
import fr.gouv.vitam.utils.FileUtil;
import fr.gouv.vitam.utils.exception.InvalidParseOperationException;
import fr.gouv.vitam.utils.exception.InvalidUuidOperationException;
import fr.gouv.vitam.utils.logging.LogbackLoggerFactory;
import fr.gouv.vitam.utils.logging.VitamLogLevel;
import fr.gouv.vitam.utils.logging.VitamLogger;
import fr.gouv.vitam.utils.logging.VitamLoggerFactory;

/**
 * Ingest file into ElasticSearch and into file, then eventually in MongoDB
 *
 * @author "Frederic Bregier"
 *
 */
public class MainPrepareIngestFile extends MainIngestFile {
    private static VitamLogger LOGGER = null;

    /**
     * Will save to a File and to ElasticSearch, then to MongoDB
     *
     * @param args
     *            logfile eraze/noeraze host database escluster unicast start nbload file fileout limitdepth nbThread mongoimport
     *            stopindex/xx
     *
     *            <ol>
     *            <li>logfile = Log4J configuration log file</li>
     *            <li>noeraze/index = index will (re)create index, else nothing is done</li>
     *            <li>host = MongoDB host</li>
     *            <li>database = MongoDB database name as VitamLinks</li>
     *            <li>escluster = ElasticSearch cluster name</li>
     *            <li>unicast = ElasticSearch unicast servers list (as in "mdb001, mdb002, mdb003")</li>
     *            <li>start = start index in the bench (will be for instance between 1-1000 start from 100)</li>
     *            <li>nbload = number of iteration from start</li>
     *            <li>file = ingest file</li>
     *            <li>fileout = output saved</li>
     *            <li>limitdepth = from which level the output is saved to the file and not to MongoDB</li>
     *            <li>nbThread = number of thread (default 1)</li>
     *            <li>mongoimport = optional command for import</li>
     *            <li>stopindex/xx = shall we stop index during import in MongoDB</li>
     *            </ol>
     */
    public static void main(final String[] args) throws Exception {
        if (args.length < 6) {
            System.err.println(
                    "need: logfile noeraze/index host database escluster unicast start nbload file fileout limitdepth mongoimport stopindex/xx nbThread");
            // System.err.println("before was need: logfile nbload files eraze/noeraze start host escluster unicast fileout limitdepth mongoimport 0/1 (1=stop index)");
            return;
        }
        final String log4j = args[0];
        PropertyConfigurator.configure(log4j);
        VitamLoggerFactory.setDefaultFactory(new LogbackLoggerFactory(VitamLogLevel.WARN));
        LOGGER = VitamLoggerFactory.getInstance(MainPrepareIngestFile.class);
        boolean reindex = false;
        if (args.length > 1) {
            reindex = args[1].equals("index");
        }
        if (simulate) {
            reindex = false;
        }
        final String networkHost = "192.168.56.102";
        GlobalDatas.localNetworkAddress = networkHost;
        // connect to the local database server
        if (args.length > 2) {
            host = args[2];
        }
        if (args.length > 3) {
            database = args[3];
        }
        if (args.length > 4) {
            esbase = args[4];
        }
        if (args.length > 5) {
            unicast = args[5];
        }
        int realnb = -1;
        if (args.length > 6) {
            startFrom = Integer.parseInt(args[6]);
        }
        if (args.length > 7) {
            realnb = Integer.parseInt(args[7]);
        }
        if (args.length > 8) {
            ingest = FileUtil.readFile(args[8]);
        }
        if (args.length > 9) {
            fileout = args[9];
        }
        if (args.length > 10) {
            final int stoplevel = Integer.parseInt(args[10]);
            minleveltofile = stoplevel;
        }
        if (args.length > 11) {
            nbThread = Integer.parseInt(args[11]);
        }
        if (args.length > 12) {
            commandMongo = args[12];
        }
        boolean stopindex = false;
        if (args.length > 13) {
            stopindex = args[13].equalsIgnoreCase("stopindex");
        }
        LOGGER.debug("Start with " + reindex + ":" + host + ":" + database + ":" + esbase + ":" + unicast);
        if (args.length > 6) {
            LOGGER.debug("and " + startFrom + ":" + realnb + ":" + ingest + ":" + fileout + ":" + minleveltofile
                    + ":" + nbThread + ":" + commandMongo + ":" + stopindex);
        }
        MongoDbAccess dbvitam = null;
        try {
            MAXTHREAD += nbThread;
            final MongoClientOptions options = new MongoClientOptions.Builder().connectionsPerHost(MAXTHREAD)
                    .build();
            mongoClient = new MongoClient(host, options);
            mongoClient.setReadPreference(ReadPreference.primaryPreferred());
            dbvitam = new MongoDbAccess(mongoClient, database, esbase, unicast, reindex);
            // get a list of the collections in this database and print them out
            LOGGER.debug(dbvitam.toString());
            if (realnb < 0) {
                return;
            }
            // drop all the data in it
            final ParserIngest parser = new ParserIngest(true);
            parser.parse(ingest);
            model = parser.getModel();
            if (reindex) {
                LOGGER.debug("ensureIndex");
                dbvitam.ensureIndex();
                if (model != null) {
                    LOGGER.debug("updateEsIndex");
                    dbvitam.updateEsIndex(model);
                }
                LOGGER.debug("end Index");
            }
            LOGGER.warn(dbvitam.toString());

            final int stepnb = realnb;
            nb = stepnb;
            loadt = new AtomicLong(0);
            cptMaip.set(0);
            runOnce(dbvitam);
        } catch (Exception e) {
            LOGGER.error(e);
        } finally {
            // release resources
            final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(2);
            final ToClean toclean = new ToClean(dbvitam);
            scheduler.schedule(toclean, 1, TimeUnit.MILLISECONDS);
            final ToShutdown toShutdown = new ToShutdown();
            scheduler.schedule(toShutdown, 5000, TimeUnit.MILLISECONDS);
            scheduler.awaitTermination(7000, TimeUnit.MILLISECONDS);
            System.exit(0);
        }

    }

    private static final void runOnce(final MongoDbAccess dbvitam)
            throws InterruptedException, InstantiationException, IllegalAccessException, IOException {
        MainPrepareIngestFile ingests = null;
        nb = nb / nbThread;
        int step = startFrom;
        final int interval = nb;
        LOGGER.warn("Load starting... " + nbThread + ":" + nb + ":" + interval);
        for (int i = 0; i < nbThread; i++) {
            ingests = new MainPrepareIngestFile();
            ingests.start = step;
            ingests.stop = step + interval - 1;
            step += interval;
            ingests.run();
        }
        System.out.println("Load ended");
        final FileOutputStream outputStream = new FileOutputStream(fileout);
        for (DAip daip : ParserIngest.savedDaips.values()) {
            daip = DAip.findOne(dbvitam, daip.getId());
            //daip.load(dbvitam);
            daip.toFile(outputStream);
        }
        outputStream.close();

        final FileOutputStream outputStreamDomain = new FileOutputStream(fileout + ".domain.json");
        Domain domain = null;
        DBCursor cursor = dbvitam.domains.collection.find();
        while (cursor.hasNext()) {
            domain = (Domain) cursor.next();
            String toprint = domain.toStringDirect() + "\n";
            try {
                outputStreamDomain.write(toprint.getBytes());
            } catch (final IOException e) {
                LOGGER.error("Cannot save to File", e);
            }
            toprint = null;
        }
        outputStreamDomain.close();

        /*
         * System.out.println("All elements\n================================================================");
         * DbVitam.printStructure(dbvitam);
         */
        final long nbBigM = dbvitam.getDaipSize();
        final long nbBigD = dbvitam.getPaipSize();
        System.out.println("\n Big Test (" + nbThread + " Threads chacune " + nb + " itrations de load, nb MAIP: "
                + cptMaip.get() + ") with MAIP: " + nbBigM + " DATA: " + nbBigD + " => Load:"
                + (loadt.get()) / ((float) cptMaip.get()));

        System.out.println("\nThread;nbLoad;nbTotal;Load");
        System.out.println(
                nbThread + ";" + cptMaip.get() + ";" + nbBigM + ";" + (loadt.get()) / ((float) cptMaip.get()));
    }

    @Override
    public void run() {
        MongoDbAccess dbvitam = null;
        try {
            dbvitam = new MongoDbAccess(mongoClient, database, esbase, unicast, false);
            // now ingest metaaip/metafield/data
            final long date11 = System.currentTimeMillis();
            parserIngest = new ParserIngest(simulate);

            final String filename = fileout + "-" + start + "-" + stop + ".json";
            final File file = new File(filename);
            final FileOutputStream outputStream = new FileOutputStream(file);
            parserIngest.bufferedOutputStream = new BufferedOutputStream(outputStream);
            System.out.println("Start to File: " + filename);
            parserIngest.parse(ingest);
            parserIngest.executeToFile(dbvitam, start, stop, false);
            final long date12 = System.currentTimeMillis();
            loadt.addAndGet(date12 - date11);
            cptMaip.addAndGet(parserIngest.getTotalCount());
            files.add(file);
            return;
        } catch (final InvalidExecOperationException e) {
            LOGGER.error(e);
        } catch (final InvalidParseOperationException e) {
            LOGGER.error(e);
        } catch (final InvalidUuidOperationException e) {
            LOGGER.error(e);
        } catch (final FileNotFoundException e) {
            LOGGER.error(e);
        } finally {
            // release resources
            if (parserIngest.bufferedOutputStream != null) {
                if (GlobalDatas.PRINT_REQUEST) {
                    System.out.println("Flushing file");
                }
                try {
                    parserIngest.bufferedOutputStream.flush();
                } catch (final IOException e) {
                }
                if (GlobalDatas.PRINT_REQUEST) {
                    System.out.println("Closing file");
                }
                try {
                    parserIngest.bufferedOutputStream.close();
                } catch (final IOException e) {
                }
                if (GlobalDatas.PRINT_REQUEST) {
                    System.out.println("File closed");
                }
            }
            if (dbvitam != null) {
                dbvitam.close();
            }
        }
    }
}