org.codice.ddf.commands.catalog.DumpCommand.java Source code

Java tutorial

Introduction

Here is the source code for org.codice.ddf.commands.catalog.DumpCommand.java

Source

/**
 * Copyright (c) Codice Foundation
 * <p>
 * This is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser
 * General Public License as published by the Free Software Foundation, either version 3 of the
 * License, or any later version.
 * <p>
 * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
 * even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details. A copy of the GNU Lesser General Public License
 * is distributed along with this program and can be found at
 * <http://www.gnu.org/licenses/lgpl.html>.
 */
package org.codice.ddf.commands.catalog;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;

import org.apache.commons.io.FileUtils;
import org.apache.felix.gogo.commands.Argument;
import org.apache.felix.gogo.commands.Command;
import org.apache.felix.gogo.commands.Option;
import org.codice.ddf.commands.catalog.facade.CatalogFacade;
import org.geotools.filter.text.cql2.CQL;
import org.joda.time.DateTime;
import org.joda.time.Period;
import org.joda.time.format.PeriodFormatter;
import org.joda.time.format.PeriodFormatterBuilder;
import org.opengis.filter.Filter;
import org.osgi.framework.BundleContext;
import org.osgi.framework.InvalidSyntaxException;
import org.osgi.framework.ServiceReference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import ddf.catalog.Constants;
import ddf.catalog.data.BinaryContent;
import ddf.catalog.data.Metacard;
import ddf.catalog.data.Result;
import ddf.catalog.data.impl.MetacardImpl;
import ddf.catalog.filter.FilterBuilder;
import ddf.catalog.operation.SourceResponse;
import ddf.catalog.operation.impl.QueryImpl;
import ddf.catalog.operation.impl.QueryRequestImpl;
import ddf.catalog.transform.CatalogTransformerException;
import ddf.catalog.transform.MetacardTransformer;

@Command(scope = CatalogCommands.NAMESPACE, name = "dump", description = "Exports Metacards from the current Catalog. Does not remove them.\n\tDate filters are ANDed together, and are exclusive for range.\n\tISO8601 format includes YYYY-MM-dd, YYYY-MM-ddTHH, YYYY-MM-ddTHH:mm, YYYY-MM-ddTHH:mm:ss, YYY-MM-ddTHH:mm:ss.sss, THH:mm:sss. See documentation for full syntax and examples.")
public class DumpCommand extends CatalogCommands {

    private static final Logger LOGGER = LoggerFactory.getLogger(DumpCommand.class);

    private static List<MetacardTransformer> transformers = null;

    private final PeriodFormatter timeFormatter = new PeriodFormatterBuilder().printZeroRarelyLast().appendDays()
            .appendSuffix(" day", " days").appendSeparator(" ").appendHours().appendSuffix(" hour", " hours")
            .appendSeparator(" ").appendMinutes().appendSuffix(" minute", " minutes").appendSeparator(" ")
            .appendSeconds().appendSuffix(" second", " seconds").toFormatter();

    @Argument(name = "Dump directory path", description = "Directory to export Metacards into. Paths are absolute and must be in quotes.  Files in directory will be overwritten if they already exist.", index = 0, multiValued = false, required = true)
    String dirPath = null;

    @Argument(name = "Batch size", description = "Number of Metacards to retrieve and export at a time until completion. Change this argument based on system memory and CatalogProvider limits.", index = 1, multiValued = false, required = false)
    int pageSize = 1000;

    // DDF-535: remove "Transformer" alias in DDF 3.0
    @Option(name = "--transformer", required = false, aliases = { "-t",
            "Transformer" }, multiValued = false, description = "The metacard transformer ID to use to transform metacards into data files. The default metacard transformer is the Java serialization transformer.")
    String transformerId = DEFAULT_TRANSFORMER_ID;

    // DDF-535: remove "Extension" alias in DDF 3.0
    @Option(name = "--extension", required = false, aliases = { "-e",
            "Extension" }, multiValued = false, description = "The file extension of the data files.")
    String fileExtension = null;

    @Option(name = "--created-after", required = false, aliases = {
            "-ca" }, multiValued = false, description = "Include only entries created after this date/time (ISO8601 format).")
    String createdAfter = null;

    @Option(name = "--created-before", required = false, aliases = {
            "-cb" }, multiValued = false, description = "Include only entries created before this date/time (ISO8601 format).")
    String createdBefore = null;

    @Option(name = "--modified-after", required = false, aliases = {
            "-ma" }, multiValued = false, description = "Include only entries modified after this date/time (ISO8601 format).")
    String modifiedAfter = null;

    @Option(name = "--modified-before", required = false, aliases = {
            "-mb" }, multiValued = false, description = "Include only entries modified before this date/time (ISO8601 format)")
    String modifiedBefore = null;

    @Option(name = "--cql", required = false, aliases = {}, multiValued = false, description = "Search using CQL Filter expressions.\n"
            + "CQL Examples:\n" + "\tTextual:   search --cql \"title like 'some text'\"\n"
            + "\tTemporal:  search --cql \"modified before 2012-09-01T12:30:00Z\"\n"
            + "\tSpatial:   search --cql \"DWITHIN(location, POINT (1 2) , 10, kilometers)\"\n"
            + "\tComplex:   search --cql \"title like 'some text' AND modified before 2012-09-01T12:30:00Z\"")
    String cqlFilter = null;

    @Option(name = "--multithreaded", required = false, aliases = { "-m",
            "Multithreaded" }, multiValued = false, description = "Number of threads to use when dumping. Setting "
                    + "this value too high for your system can cause performance degradation.")
    int multithreaded = 20;

    @Option(name = "--dirlevel", required = false, multiValued = false, description = "Number of subdirectory levels to create.  Two characters from the ID "
            + "will be used to name each subdirectory level.")
    int dirLevel = 0;

    @Override
    protected Object executeWithSubject() throws Exception {
        final File dumpDir = new File(dirPath);

        if (!dumpDir.exists()) {
            printErrorMessage("Directory [" + dirPath + "] must exist.");
            console.println("If the directory does indeed exist, try putting the path in quotes.");
            return null;
        }

        if (!dumpDir.isDirectory()) {
            printErrorMessage("Path [" + dirPath + "] must be a directory.");
            return null;
        }

        if (!DEFAULT_TRANSFORMER_ID.matches(transformerId)) {
            transformers = getTransformers();
            if (transformers == null) {
                console.println(transformerId + " is an invalid metacard transformer.");
                return null;
            }
        }

        CatalogFacade catalog = getCatalog();
        FilterBuilder builder = getFilterBuilder();

        Filter createdFilter = null;
        if ((createdAfter != null) && (createdBefore != null)) {
            DateTime createStartDateTime = DateTime.parse(createdAfter);
            DateTime createEndDateTime = DateTime.parse(createdBefore);
            createdFilter = builder.attribute(Metacard.CREATED).is().during().dates(createStartDateTime.toDate(),
                    createEndDateTime.toDate());
        } else if (createdAfter != null) {
            DateTime createStartDateTime = DateTime.parse(createdAfter);
            createdFilter = builder.attribute(Metacard.CREATED).is().after().date(createStartDateTime.toDate());
        } else if (createdBefore != null) {
            DateTime createEndDateTime = DateTime.parse(createdBefore);
            createdFilter = builder.attribute(Metacard.CREATED).is().before().date(createEndDateTime.toDate());
        }

        Filter modifiedFilter = null;
        if ((modifiedAfter != null) && (modifiedBefore != null)) {
            DateTime modifiedStartDateTime = DateTime.parse(modifiedAfter);
            DateTime modifiedEndDateTime = DateTime.parse(modifiedBefore);
            modifiedFilter = builder.attribute(Metacard.MODIFIED).is().during()
                    .dates(modifiedStartDateTime.toDate(), modifiedEndDateTime.toDate());
        } else if (modifiedAfter != null) {
            DateTime modifiedStartDateTime = DateTime.parse(modifiedAfter);
            modifiedFilter = builder.attribute(Metacard.MODIFIED).is().after().date(modifiedStartDateTime.toDate());
        } else if (modifiedBefore != null) {
            DateTime modifiedEndDateTime = DateTime.parse(modifiedBefore);
            modifiedFilter = builder.attribute(Metacard.MODIFIED).is().before().date(modifiedEndDateTime.toDate());
        }

        Filter filter = null;
        if ((createdFilter != null) && (modifiedFilter != null)) {
            // Filter by both created and modified dates
            filter = builder.allOf(createdFilter, modifiedFilter);
        } else if (createdFilter != null) {
            // Only filter by created date
            filter = createdFilter;
        } else if (modifiedFilter != null) {
            // Only filter by modified date
            filter = modifiedFilter;
        } else {
            // Don't filter by date range
            filter = builder.attribute(Metacard.ID).is().like().text(WILDCARD);
        }

        if (cqlFilter != null) {
            filter = CQL.toFilter(cqlFilter);
        }

        QueryImpl query = new QueryImpl(filter);
        query.setRequestsTotalResultsCount(false);
        query.setPageSize(pageSize);

        Map<String, Serializable> props = new HashMap<String, Serializable>();
        // Avoid caching all results while dumping with native query mode
        props.put("mode", "native");

        final AtomicLong resultCount = new AtomicLong(0);
        long start = System.currentTimeMillis();

        SourceResponse response = catalog.query(new QueryRequestImpl(query, props));

        BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<Runnable>(multithreaded);
        RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy();
        final ExecutorService executorService = new ThreadPoolExecutor(multithreaded, multithreaded, 0L,
                TimeUnit.MILLISECONDS, blockingQueue, rejectedExecutionHandler);

        while (response.getResults().size() > 0) {
            response = catalog.query(new QueryRequestImpl(query, props));

            if (multithreaded > 1) {
                final List<Result> results = new ArrayList<Result>(response.getResults());
                executorService.submit(new Runnable() {
                    @Override
                    public void run() {
                        boolean transformationFailed = false;
                        for (final Result result : results) {
                            Metacard metacard = result.getMetacard();
                            try {
                                exportMetacard(dumpDir, metacard);
                            } catch (IOException | CatalogTransformerException e) {
                                transformationFailed = true;
                                LOGGER.debug("Failed to dump metacard {}", metacard.getId(), e);
                                executorService.shutdownNow();
                            }
                            printStatus(resultCount.incrementAndGet());
                        }
                        if (transformationFailed) {
                            LOGGER.error(
                                    "One or more metacards failed to transform. Enable debug log for more details.");
                        }
                    }
                });
            } else {
                for (final Result result : response.getResults()) {
                    Metacard metacard = result.getMetacard();
                    exportMetacard(dumpDir, metacard);
                    printStatus(resultCount.incrementAndGet());
                }
            }

            if (response.getResults().size() < pageSize || pageSize == -1) {
                break;
            }

            if (pageSize > 0) {
                query.setStartIndex(query.getStartIndex() + pageSize);
            }
        }

        executorService.shutdown();

        while (!executorService.isTerminated()) {
            try {
                TimeUnit.MILLISECONDS.sleep(100);
            } catch (InterruptedException e) {
                // ignore
            }
        }

        long end = System.currentTimeMillis();
        String elapsedTime = timeFormatter.print(new Period(start, end).withMillis(0));
        console.printf(" %d file(s) dumped in %s\t%n", resultCount.get(), elapsedTime);
        LOGGER.info("{} file(s) dumped in {}", resultCount.get(), elapsedTime);
        console.println();

        return null;
    }

    private void exportMetacard(File dumpLocation, Metacard metacard)
            throws IOException, CatalogTransformerException {

        if (DEFAULT_TRANSFORMER_ID.matches(transformerId)) {
            try (ObjectOutputStream oos = new ObjectOutputStream(
                    new FileOutputStream(getOutputFile(dumpLocation, metacard)))) {
                oos.writeObject(new MetacardImpl(metacard));
                oos.flush();
            }
        } else {

            BinaryContent binaryContent;
            if (metacard != null) {
                try (FileOutputStream fos = new FileOutputStream(getOutputFile(dumpLocation, metacard))) {
                    for (MetacardTransformer transformer : transformers) {
                        binaryContent = transformer.transform(metacard, new HashMap<>());
                        if (binaryContent != null) {
                            fos.write(binaryContent.getByteArray());
                            break;
                        }
                    }
                }
            }
        }
    }

    private File getOutputFile(File dumpLocation, Metacard metacard) throws IOException {
        String extension = "";
        if (fileExtension != null) {
            extension = "." + fileExtension;
        }

        String id = metacard.getId();
        File parent = dumpLocation;

        if (dirLevel > 0 && id.length() >= dirLevel * 2) {
            for (int i = 0; i < dirLevel; i++) {
                parent = new File(parent, id.substring(i * 2, i * 2 + 2));
            }
            FileUtils.forceMkdir(parent);
        }

        return new File(parent, id + extension);
    }

    protected void printStatus(long count) {
        console.print(String.format(" %d file(s) dumped\t\r", count));
        console.flush();
    }

    private List<MetacardTransformer> getTransformers() {

        BundleContext bundleContext = getBundleContext();
        ServiceReference[] refs = null;
        try {
            refs = bundleContext.getAllServiceReferences(MetacardTransformer.class.getName(),
                    "(|" + "(" + Constants.SERVICE_ID + "=" + transformerId + ")" + ")");

        } catch (InvalidSyntaxException e) {
            console.printf("Fail to get MetacardTransformer references due to %s", e.getMessage());
        }
        if (refs == null || refs.length == 0) {
            return null;
        }

        List<MetacardTransformer> metacardTransformerList = new ArrayList<MetacardTransformer>();
        for (int i = 0; i < refs.length; i++) {

            metacardTransformerList.add((MetacardTransformer) bundleContext.getService(refs[i]));
        }

        return metacardTransformerList;
    }

}