dk.dma.ais.store.FileExportRest.java Source code

Java tutorial

Introduction

Here is the source code for dk.dma.ais.store.FileExportRest.java

Source

/* Copyright (c) 2011 Danish Maritime Authority.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package dk.dma.ais.store;

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URLEncoder;
import java.security.MessageDigest;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;

import javax.xml.bind.annotation.adapters.HexBinaryAdapter;

import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.Interval;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;

import com.beust.jcommander.Parameter;
import com.google.inject.Injector;

import dk.dma.ais.message.AisMessage;
import dk.dma.ais.packet.AisPacket;
import dk.dma.ais.packet.AisPacketFilters;
import dk.dma.ais.packet.AisPacketOutputSinks;
import dk.dma.ais.reader.AisReader;
import dk.dma.ais.reader.AisReaders;
import dk.dma.commons.app.AbstractCommandLineTool;
import dk.dma.commons.util.io.OutputStreamSink;

/**
 * @author Jens Tuxen
 * @author David Andersen Camre
 */
public class FileExportRest extends AbstractCommandLineTool {

    private FileOutputStream fileOutputStream;
    private OutputStream outputStream;
    private OutputStreamSink<AisPacket> sink;
    private AtomicLong counter;

    // Status Variables
    long timeStart = System.currentTimeMillis();

    DecimalFormat decimalFormatter = new DecimalFormat("0.00");

    // Meta data
    private long currentTimeStamp;
    private Long packageCount = 0L;
    private long lastFlushTimestamp;

    private long intervalStartTime;

    Interval intervalVar;

    private long lastLoadedTimestamp;

    private String metaFileName;

    String printPrefix = "[AIS-STORE] ";

    /** A date time formatter for utc. */
    DateTimeFormatter dateTimeFormatter = DateTimeFormat.forPattern("yyyy-M-d'T'HH:mm:ss'Z");

    @Parameter(names = "-mmsi", description = "Extract from mmsi schema")
    List<Integer> mmsis = new ArrayList<Integer>();

    @Parameter(names = "-filter", description = "The filter to apply")
    String filter;

    @Parameter(names = "-interval", description = "The ISO 8601 time interval for data to export")
    String interval;

    @Parameter(names = { "-area", "-box",
            "-geo" }, description = "Extract from geopgraphic cells schema within bounding box lat1,lon1,lat2,lon2")
    String area;

    @Parameter(names = "-outputFormat", description = "Output format, options: raw, json, jsonObject (use -columns), kml, kmz, table (use -columns)")
    String outputFormat = "raw";

    @Parameter(names = "-columns", description = "Optional columns, used for jsonObject and table.")
    String columns;

    @Parameter(names = "-separator", description = "Optional separator, used for table format.")
    String separator = ",";

    @Parameter(names = { "-file", "-output", "-o" }, description = "File to extract to (default is stdout)")
    String filePath;

    @Parameter(names = { "-fileName" }, description = "File to extract to (default is stdout)")
    String fileName;

    @Parameter(names = "-fetchSize", description = "internal fetch size buffer")
    Integer fetchSize = -1;

    @Parameter(names = "-force", description = "Disregard existing download and force redownload")
    boolean forceDownload = false;

    /** {@inheritDoc} */
    @Override
    protected void run(Injector injector) throws Exception {
        printAisStoreNL("AIS STORE COMMAND LINE TOOL INITIATED");
        printAisStoreLine();

        // Hardcoded values
        // interval = "2015-1-5T14:00:00Z/2015-1-5T14:10:00Z";
        // java -jar ais-store-cli-0.3-SNAPSHOT.jar export -area 15,-18,-10,14 -filter
        // "m.country=DNK & t.pos within bbox(15,-18,-10,14) & (t.lat<-0.3|t.lat>0.3) & (t.lon<-0.3|t.lon>0.3)" -fetchSize 30000
        // -interval
        // java -jar ais-store-cli-0.3-SNAPSHOT.jar export -area 15,-18,-10,14 -filter
        // "m.country=DNK & t.pos within bbox(15,-18,-10,14) & (t.lat<-0.3|t.lat>0.3) & (t.lon<-0.3|t.lon>0.3)" -fetchSize 30000
        // -interval 2015-1-5T14:00:00Z/2015-1-5T14:10:00Z

        // Create request
        String request = "";

        if (interval == null || interval.equals("")) {
            printAisStoreNL("No Interval provided, please check your request.");

            // return;
            terminateAndPrintHelp();
        }

        try {
            intervalVar = Interval.parse(interval);
        } catch (Exception e) {
            printAisStoreNL("Invalid Interval provided, please check your request.");
            terminateAndPrintHelp();
        }

        // intervalVar = DateTimeUtil.toInterval(intervalStr);
        intervalStartTime = intervalVar.getStartMillis();
        String intervalStr = interval.toString();

        request = request + "?interval=" + interval;

        // System.out.println("Interval parsed correct " + intervalStr);

        // Check if interval is valid, throw exception etc

        // Create task for exception throwing if args are required
        // If error, throw exception of error description then -help

        if (mmsis.size() > 0) {
            request = request + "&mmsi=";

            for (int i = 0; i < mmsis.size() - 1; i++) {
                request = request + Integer.toString(mmsis.get(i)) + ",";
            }

            request = request + mmsis.get(mmsis.size() - 1);
        }

        // filter
        // Google URL Encoder
        // "t.name like H* & t.name like HAMLET"
        // CHeck if url filter is valid, then url encode it and add to request

        // filter = "t.name like H* & t.name like HAMLET";
        // filter = "s.country in (DNK)";

        if (filter != null && !filter.equals("")) {
            String encodedFilter = URLEncoder.encode(filter, "UTF-8");

            try {
                AisPacketFilters.parseExpressionFilter(filter);
            } catch (Exception e) {
                printAisStoreNL("Invalid filter expression");
                terminateAndPrintHelp();
            }

            request = request + "&filter=" + encodedFilter;
        }

        // area
        // "&box=lat1,lon1,lat2,lon2"
        // blabla check if valid

        if (area != null && !area.equals("")) {
            request = request + "&box=" + area;
        }

        // If table, make sure column is added
        if ((columns == null || columns.equals(""))
                && (outputFormat.equals("table") || outputFormat.equals("jsonObject"))) {
            printAisStoreNL("When using outputFormat " + outputFormat + ", columns are required");
            terminateAndPrintHelp();
        }

        try {
            sink = AisPacketOutputSinks.getOutputSink(outputFormat, columns, separator);

            request = request + "&outputFormat=" + outputFormat;
        } catch (Exception e) {
            printAisStoreNL("Invalid output format provided, " + outputFormat + ", please check your request.");
            terminateAndPrintHelp();
        }

        // if table do shit
        // columns \/ REQUIRED
        // "columns=mmsi;time;timestamp"

        // Check if valid
        // Split on ";"
        // "columns=<listElement0>:list1"

        if (columns != null) {
            request = request + "&columns=" + columns;
        }

        // seperator
        // "seperator=\t"
        // Url encode and add
        if (separator != null || !separator.equals("")) {
            String encodedSeparator = URLEncoder.encode(separator, "UTF-8");
            request = request + "&seperator=" + encodedSeparator;
        }

        // fetchSize
        if (fetchSize != -1) {
            request = request + "&fetchsize=" + fetchSize;
        }

        // Get path from request, if none it will store in root of ais store client
        // filePath = "C:\\AisStoreData\\";

        if (filePath == null) {
            filePath = "";
        } else {
            filePath = filePath + "/";
        }

        // No filename provided, generate unique based on request parameters
        if (fileName == null || fileName.equals("")) {
            MessageDigest md5 = MessageDigest.getInstance("MD5");
            String hex = (new HexBinaryAdapter()).marshal(md5.digest(request.getBytes()));
            fileName = hex;
        }

        // Generate unique hashsum based on request
        metaFileName = fileName + ".aisstore";

        // boolean isTryResume = true;

        // If we are trying to resume, don't override previous file

        try {
            fileOutputStream = new FileOutputStream(filePath + fileName, !forceDownload);
        } catch (Exception e) {
            printAisStoreNL("Error occuring writing to disk, make sure the folder path exists ");
            terminateAndPrintHelp();
        }

        outputStream = new BufferedOutputStream(fileOutputStream);

        // Should we resume anything
        // We have read the file
        // If the file exists that means a previous transaction has been done

        // Do we resume, if we do, we need to find the resume point ie move the start interval

        /**
         * System.out.println("Test Compare"); System.out.println(intervalStr);
         * 
         * DateTime time = new DateTime(interval.getStartMillis(), DateTimeZone.UTC); DateTime time2 = new
         * DateTime(interval.getEndMillis(), DateTimeZone.UTC);
         * 
         * String newIntervalStr = dateTimeFormatter.withZoneUTC().print(time) + "/" + dateTimeFormatter.withZoneUTC().print(time2);
         * System.out.println(newIntervalStr); // DateTime dateTime =
         * dateTimeFormatter.parseDateTime("15-Oct-2013 11:34:26 AM").withZone(DateTimeZone.UTC);
         * 
         * // System.out.println(dateTime);
         * 
         * // Interval var = Interval.parse(intervalStr); // String dateStr = formatter.withZone(DateTimeZone.UTC).print(dateTime1);
         * //
         * 
         * System.exit(0);
         **/

        printAisStoreNL("Request generation complete.");
        printAisStoreNL("AIS Data will be saved to " + filePath + fileName);
        // System.out.println("--------------------------------------------------------------------------------");

        // We are resuming, insert a Carriage Return Line Feed
        if (!forceDownload) {

            // Load the meta data in
            readMeta();

            // We have processed some packages already
            if (packageCount != 0) {

                String str = "\r\n";
                outputStream.write(str.getBytes());
                printAisStoreLine();
                printAisStoreNL("Resume detected - Updating Request");
                // System.out.println("From " + intervalStr);

                // Update intervalStr
                DateTime time = new DateTime(lastLoadedTimestamp, DateTimeZone.UTC);
                DateTime time2 = new DateTime(intervalVar.getEndMillis(), DateTimeZone.UTC);

                intervalStr = dateTimeFormatter.withZoneUTC().print(time) + "/"
                        + dateTimeFormatter.withZoneUTC().print(time2);
                // System.out.println("To " + intervalStr);
                printAisStoreLine();

                // System.out.println("The last stored timestamp was \n" + lastLoadedTimestamp);
                // Interval interval2 = DateTimeUtil.toInterval(intervalStr);
                // System.out.println(interval2.getStartMillis());

            } else {
                writeMetaInit(intervalVar.getStartMillis());
                lastLoadedTimestamp = intervalVar.getStartMillis();
            }
        } else {
            // We are starting a new request, create a new meta init
            writeMetaInit(intervalVar.getStartMillis());
            lastLoadedTimestamp = intervalVar.getStartMillis();

        }
        // System.out.println("Interval Str is " + intervalStr);
        // System.exit(0);

        // Initialize
        counter = new AtomicLong(packageCount);

        // Do we need to set a new interval start based on the meta data read?

        DefaultHttpClient httpClient = new DefaultHttpClient();

        HttpHost target = new HttpHost("ais2.e-navigation.net", 443, "https");

        request = "/aisview/rest/store/query" + request;

        HttpGet getRequest = new HttpGet(request);
        // HttpGet getRequest = new
        // HttpGet("/aisview/rest/store/query?interval=2015-1-1T10:00:00Z/2015-2-1T10:10:00Z&box=65.145,-5.373,34.450,76.893");
        // + "&mmsi=219230000"
        // + "&mmsi=219230000"
        printAisStoreNL("Executing request to " + target);
        printAisStoreNL("Request is: " + request);

        HttpResponse httpResponse = httpClient.execute(target, getRequest);
        HttpEntity entity = httpResponse.getEntity();

        // Check we have an OK from server etc.

        printAisStoreLine();

        boolean terminateFailure = false;

        StatusLine reply = httpResponse.getStatusLine();
        switch (reply.getStatusCode()) {
        case HttpStatus.SC_OK:
            printAisStoreNL("Server Accepted Connection, download will begin shortly");
            printAisStoreLine();
            break;
        default:
            printAisStoreNL("An error occured establishing connection to the server. ");
            printAisStoreNL(
                    "Server returned Status Code " + reply.getStatusCode() + " with " + reply.getReasonPhrase());
            terminateFailure = true;
            break;
        }

        if (terminateFailure) {
            return;
        }

        // System.out.println("Got reply " + reply.getReasonPhrase() + " status code " + reply.getStatusCode());

        // String httpServerReply = httpResponse.getStatusLine();
        // System.out.println(httpResponse.getStatusLine());
        //
        // Header[] headers = httpResponse.getAllHeaders();
        // for (int i = 0; i < headers.length; i++) {
        // System.out.println(headers[i]);
        // }

        // Do we use the footer?

        AisReader aisReader;

        if (entity != null) {
            InputStream inputStream = entity.getContent();

            aisReader = aisReadWriter(inputStream);

            aisReader.start();
            aisReader.join();

            // Write the remainder still stored in buffer, update the final meta data with the finished data
            writeMetaUpdate(currentTimeStamp, counter.get());

            // Write the footer
            sink.footer(outputStream, counter.get());

            // Closer and flush the buffer
            outputStream.flush();
            outputStream.close();

            // Close and flush the file stream
            fileOutputStream.flush();
            fileOutputStream.close();
        }

        // print a new line to move on from previous /r

        printAisStoreNL(
                "Downloading AIS Data 100% Estimated Time Left: 00:00:00                                       ");
        printAisStoreLine();
        printAisStoreNL("DOWNLOAD SUCCESS");
        printAisStoreLine();

        // We know current time
        long currentTime = System.currentTimeMillis();
        // How long have we been running
        long millis = currentTime - timeStart;
        String timeLeftStr = String.format("%02d:%02d:%02d", TimeUnit.MILLISECONDS.toHours(millis),
                TimeUnit.MILLISECONDS.toMinutes(millis)
                        - TimeUnit.HOURS.toMinutes(TimeUnit.MILLISECONDS.toHours(millis)),
                TimeUnit.MILLISECONDS.toSeconds(millis)
                        - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(millis)));

        printAisStoreNL("Total Time " + timeLeftStr);
        printAisStoreNL("Finished at: " + new Date());
        printAisStoreNL("Messages recieved " + counter);

        // printAisStore("Query took " + timeLeftStr);
    }

    private void printDownloadStatus() {
        // Determine how far left we have to go

        // We know current time
        long currentTime = System.currentTimeMillis();
        // How long have we been running
        long milisecondsRunning = currentTime - timeStart;

        // Calculate total % done:

        // Total AIS time to process in miliseconds
        long aisTimeTotalOriginal = intervalVar.getEndMillis() - intervalStartTime;

        // Stretch of processed AIS time:
        long processedAisTimeOriginal = lastFlushTimestamp - intervalStartTime;

        double percentDoneOriginal = (double) processedAisTimeOriginal / (double) aisTimeTotalOriginal * 100;

        // Calculate the estimated time

        // Total AIS time to process in miliseconds
        long aisTimeTotal = intervalVar.getEndMillis() - lastLoadedTimestamp;

        // Stretch of processed AIS time:
        long processedAisTime = lastFlushTimestamp - lastLoadedTimestamp;
        double percentDoneNow = (double) processedAisTime / (double) aisTimeTotal * 100;

        double goToPercent = (100 - percentDoneNow);

        // How many % do we calculate pr. milisecond
        double percentPrMilisecond = percentDoneNow / ((double) milisecondsRunning);
        double timeLeft = goToPercent / percentPrMilisecond;

        long millis = (long) timeLeft;

        String timeLeftStr = String.format("%02d:%02d:%02d", TimeUnit.MILLISECONDS.toHours(millis),
                TimeUnit.MILLISECONDS.toMinutes(millis)
                        - TimeUnit.HOURS.toMinutes(TimeUnit.MILLISECONDS.toHours(millis)),
                TimeUnit.MILLISECONDS.toSeconds(millis)
                        - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(millis)));

        String percentDoneStr = decimalFormatter.format(percentDoneOriginal) + "%";
        // String percentDoneStr = ((double) ((int) (percentDoneOriginal * 100))) / 100 + "%";

        // String part1DownloadMessage = "Downloading AIS Data " + percentDoneStr;
        // String part2DownloadMessage = " Estimated Time Left: " + timeLeftStr;

        // if (part1DownloadMessage.length() < )

        printAisStore("Downloading AIS Data " + percentDoneStr + " Estimated Time Left: " + timeLeftStr + "\r");
        // printAisStore();
        // int seconds = (int) (timeLeft / 1000) % 60;
        // int minutes = (int) ((timeLeft / (1000 * 60)) % 60);
        // int hours = (int) ((timeLeft / (1000 * 60 * 60)) % 24);
        // System.out.println(hours + ":" + minutes + ":" + seconds);

        // System.out.println("Miliseconds running" + milisecondsRunning);

        //
        // // AIS Time left of request
        // long aisTimeLeft = intervalVar.getEndMillis() - lastFlushTimestamp;
        //
        // // How long have we spent parsing up to now
        // long aisTimeParsed = processedAisTime / milisecondsRunning;
        //
        // System.out.println("We have in " + milisecondsRunning + " miliseconds processed " + processedAisTime + " AIS Interval");
        // System.out.println("AIS Time Left: " + aisTimeLeft);
        // System.out.println("aisTimeParsed " + aisTimeParsed);

        // counterCurrent.get();

        //

        // If we are resuming our % will be further along

        // We know our count since start
        // We know how long we have been running

    }

    /**
     * Read the meta file containing data regarding previous transactions in
     * 
     * The meta file will have the following fields: file name - the file containing the data, if no filename is provided in request
     * we use hash of request to ensure uniqueness last time stamp - the last recorded timestamp packageCount - the amount of
     * messages written, used in footer
     * 
     * @param path
     * @param metaFileName
     */
    private void readMeta() throws IOException, ParseException {

        File f = new File(filePath + metaFileName);
        if (f.exists() && !f.isDirectory()) {

            JSONParser parser = new JSONParser();

            try {

                Object obj = parser.parse(new FileReader(filePath + metaFileName));

                JSONObject jsonObject = (JSONObject) obj;

                // if (jsonObject.get("filename") != null) {
                fileName = (String) jsonObject.get("filename");
                // System.out.println(fileName);

                // }

                lastLoadedTimestamp = (Long) jsonObject.get("timestamp");
                // System.out.println(lastFlushTimestamp);

                packageCount = (Long) jsonObject.get("packageCount");

                // System.out.println(packageCount);

            }

            catch (Exception e) {
                e.printStackTrace();
                System.out.println("No meta file detected or invalid meta file");
            }
        }
    }

    @SuppressWarnings("unchecked")
    private void writeMetaInit(Long startInterval) {
        JSONObject obj = new JSONObject();
        obj.put("filename", fileName);
        obj.put("timestamp", startInterval);
        obj.put("packageCount", packageCount);

        try {
            // System.out.println("Writing");
            FileWriter file = new FileWriter(filePath + metaFileName);
            file.write(obj.toJSONString());
            file.flush();
            file.close();

        } catch (IOException e) {
            e.printStackTrace();
        }

    }

    @SuppressWarnings("unchecked")
    private void writeMetaUpdate(long timestamp, long packageCount) {

        // System.out.println("Updating meta!");
        FileWriter file = null;
        JSONObject obj = new JSONObject();
        obj.put("filename", fileName);
        obj.put("timestamp", timestamp);
        obj.put("packageCount", new Long(packageCount));

        try {

            file = new FileWriter(filePath + metaFileName);
            file.write(obj.toJSONString());
            file.flush();

        } catch (IOException e) {
            // e.printStackTrace();
            System.out.println("Failed to write because reasons " + e.getMessage());
        } finally {
            try {
                file.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
                System.out.println("Failed to close file! " + e.getMessage());
            }
        }
    }

    private AisReader aisReadWriter(InputStream in) throws Exception {

        AisReader r = AisReaders.createReaderFromInputStream(in);
        r.registerPacketHandler(new Consumer<AisPacket>() {

            @Override
            public void accept(AisPacket t) {
                AisMessage message = t.tryGetAisMessage();

                // System.out.println(message.toString());

                currentTimeStamp = t.getBestTimestamp();

                if (lastLoadedTimestamp >= currentTimeStamp) {
                    // System.out.println("Skipping Message!");
                    return;
                }

                if (message == null) {
                    return;
                }

                try {
                    sink.process(outputStream, t, counter.incrementAndGet());

                    if (currentTimeStamp != lastFlushTimestamp && counter.get() % 10000 == 0) {

                        // We have a new timestamp sequence
                        lastFlushTimestamp = currentTimeStamp;

                        writeMetaUpdate(lastFlushTimestamp, counter.get());

                        // Force flush on both

                        outputStream.flush();
                        fileOutputStream.flush();

                        // Write
                        // lastFlushTimestamp
                        // timestamp

                        // if (counter.get() >= 1000) {
                        // System.out.println("Terminating as part of test! Last written timestamp was " + lastFlushTimestamp);
                        // System.exit(0);
                        // }

                        // Update user on progress
                        printDownloadStatus();
                    }

                    //

                } catch (IOException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
            }

        });

        return r;

    }

    public static void main(String[] args) throws Exception {
        new FileExportRest().execute(args);
    }

    private void terminateAndPrintHelp() {
        System.out.println("Terminating AIS Store Command Line");

        try {
            new FileExportRest().execute(new String[] { "-help" });
        } catch (Exception e) {
        }
        System.exit(-1);
    }

    private void printAisStore(String toPrint) {
        System.out.print(printPrefix + toPrint);
    }

    private void printAisStoreNL(String toPrint) {
        System.out.println(printPrefix + toPrint);
    }

    private void printAisStoreLine() {
        System.out.println(
                printPrefix + "--------------------------------------------------------------------------------");
    }
}