dk.dma.ais.lib.FileConvert.java Source code

Java tutorial

Introduction

Here is the source code for dk.dma.ais.lib.FileConvert.java

Source

/* Copyright (c) 2011 Danish Maritime Authority.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dk.dma.ais.lib;

import com.beust.jcommander.Parameter;
import com.google.inject.Injector;
import dk.dma.ais.packet.AisPacket;
import dk.dma.ais.packet.AisPacketOutputSinks;
import dk.dma.ais.packet.AisPacketReader;
import dk.dma.commons.app.AbstractCommandLineTool;
import dk.dma.commons.util.io.OutputStreamSink;
import dk.dma.enav.util.function.EConsumer;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.lang.reflect.Field;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

/**
 * Converts a set of files into another sink format
 *
 * @author Jens Tuxen
 * @author Thomas Borg Salling
 *
 */
public class FileConvert extends AbstractCommandLineTool {

    /** The logger. */
    static final Logger LOG = LoggerFactory.getLogger(FileConvert.class);

    @Parameter(required = true, description = "files to import...")
    List<String> sources;

    /** Where files should be moved to after having been processed. */
    @Parameter(names = "-path", required = false, description = "base-path to put converted files")
    String convertTo = "converted/";

    /** Where files should be moved to after having been processed. */
    @Parameter(names = "-keepStructure", required = false, description = "Whether to keep path structure")
    boolean keepFileStructure;

    @Parameter(names = "-fileEnding", required = false, description = "File ending")
    String fileEnding;

    @Parameter(names = "-outputFormat", required = false, description = "Output formats: [OUTPUT_TO_TEXT, OUTPUT_PREFIXED_SENTENCES, OUTPUT_TO_HTML, table, csv, csv_stateful, json, jsonobject, kml, kmz]")
    String outputSinkFormat = "OUTPUT_PREFIXED_SENTENCES";

    @Parameter(names = "-columns", required = false, description = "Optional columns, required with -outputFormat table. use ; as delimiter. Example: -columns mmsi;time;lat;lon")
    String columns;

    @Parameter(names = "-kmzPrimaryMmsi", required = false, description = "For use with '-outputFormat kmz': Primary mmsi no. Affects color of vessel.")
    int kmzPrimaryMmsi = -1;

    @Parameter(names = "-kmzSecondaryMmsi", required = false, description = "For use with '-outputFormat kmz': Secondary mmsi no. Affects color of vessel.")
    int kmzSecondaryMmsi = -1;

    @Parameter(names = "-kmzSnapshotAt", required = false, description = "For use with '-outputFormat kmz': Date and time in datastream for creation of snapshot folder. [YYYY-MM-DD-hh:mm]")
    String kmzSnapshotAt;

    DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH:mm");

    public FileConvert() {
    }

    /**
     * getOutputSinks
     *
     * @return
     */
    static final List<String> getOutputSinks() {
        LinkedList<String> methods = new LinkedList<String>();
        Field[] declaredMethods = AisPacketOutputSinks.class.getDeclaredFields();
        for (Field m : declaredMethods) {
            if (m.getName().startsWith("OUTPUT")) {
                methods.add(m.getName());
            }
        }
        return methods;
    }

    /** {@inheritDoc} */
    @Override
    protected void run(Injector injector) throws Exception {
        configureFileEnding();

        final EConsumer<String> consumer = new EConsumer<String>() {

            @Override
            public void accept(String s) throws IllegalArgumentException, IllegalAccessException,
                    NoSuchFieldException, SecurityException, IOException, InterruptedException {
                Path path = Paths.get(s);
                LOG.debug("Started processing file " + path);

                Path endPath;
                if (keepFileStructure) {
                    Path relative;
                    relative = path;
                    endPath = Paths.get(Paths.get(convertTo).toString(), relative.toString());
                    new File(endPath.toString()).mkdirs();
                } else {
                    endPath = Paths.get("");
                }

                String filename = path.getFileName().toString();
                if (!filename.endsWith(fileEnding))
                    filename = FilenameUtils.removeExtension(filename) + fileEnding;
                Path filePath = Paths.get(endPath.toString(), filename);

                LOG.debug("Output File: " + filePath.toString());

                final OutputStream fos = new FileOutputStream(filePath.toString()); // 2

                final boolean createSituationFolder = !StringUtils.isBlank(kmzSnapshotAt);
                final long snapshotAtEpochMillis = createSituationFolder
                        ? LocalDateTime.parse(kmzSnapshotAt, formatter).toInstant(ZoneOffset.UTC).toEpochMilli()
                        : -1;

                OutputStreamSink<AisPacket> sink;
                if ("kmz".equals(outputSinkFormat)) {
                    //AisPacketKMZOutputSink(filter, createSituationFolder, createMovementsFolder, createTracksFolder, isPrimaryTarget, isSecondaryTarget, triggerSnapshot, snapshotDescriptionSupplier, movementInterpolationStep, supplyTitle, supplyDescription, iconHrefSupplier);
                    sink = AisPacketOutputSinks.newKmzSink(e -> true, // this.filter = e -> true;
                            createSituationFolder, // this.createSituationFolder = true;
                            true, // createMovementsFolder = true;
                            true, // this.createTracksFolder = true;
                            e -> kmzPrimaryMmsi <= 0 ? false : e.tryGetAisMessage().getUserId() == kmzPrimaryMmsi, // this.isPrimaryTarget = e -> false;
                            e -> kmzSecondaryMmsi <= 0 ? false
                                    : e.tryGetAisMessage().getUserId() == kmzSecondaryMmsi, // this.isSecondaryTarget = e -> false;
                            e -> e.getBestTimestamp() >= snapshotAtEpochMillis, // this.triggerSnapshot = e -> false;
                            () -> "Situation at " + kmzSnapshotAt, // this.snapshotDescriptionSupplier = null;
                            () -> 10, // this.title = defaultTitleSupplier;
                            () -> "description", // this.description = defaultDescriptionSupplier;
                            () -> "10", //this.movementInterpolationStep = defaultMovementInterpolationStepSupplier;
                            (shipTypeCargo, navigationalStatus) -> "" // this.iconHrefSupplier = defaultIconHrefSupplier;
                    );

                } else
                    sink = AisPacketOutputSinks.getOutputSink(outputSinkFormat, columns);

                sink.closeWhenFooterWritten();

                AisPacketReader apis = AisPacketReader.createFromFile(path, false);

                apis.writeTo(fos, sink);
                apis.close();
                fos.close();
            }
        };

        /*
         * Creates a pool of executors, 4 threads. Each thread will open a file using an aispacket reader, 10000 files can be
         * submitted to the queue, afterwards the calling thread will execute the job instead.
         */
        ThreadPoolExecutor threadpoolexecutor = new ThreadPoolExecutor(4, 4, 1, TimeUnit.SECONDS,
                new ArrayBlockingQueue<>(10000), new ThreadPoolExecutor.CallerRunsPolicy());
        for (final String s : sources) {
            threadpoolexecutor.execute(() -> {
                try {
                    consumer.accept(s);
                } catch (Exception e) {
                    e.printStackTrace();
                }

            });
        }

        threadpoolexecutor.shutdown();
        threadpoolexecutor.awaitTermination(999, TimeUnit.DAYS);
    }

    private void configureFileEnding() {
        if (StringUtils.isBlank(fileEnding)) {
            if (outputSinkFormat.startsWith("csv")) {
                fileEnding = ".csv";
            } else if (outputSinkFormat.startsWith("kmz")) {
                fileEnding = ".kmz";
            } else if (outputSinkFormat.startsWith("json")) {
                fileEnding = ".json";
            } else if (outputSinkFormat.startsWith("kml")) {
                fileEnding = ".kml";
            } else {
                fileEnding = ".txt";
            }
        }
    }

    public static void main(String[] args) throws Exception {
        FileConvert fc = new FileConvert();
        fc.execute(args);
    }
}