Java tutorial
/* Copyright (c) 2011 Danish Maritime Authority. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package dk.dma.ais.store; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.URLEncoder; import java.security.MessageDigest; import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import javax.xml.bind.annotation.adapters.HexBinaryAdapter; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.StatusLine; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.DefaultHttpClient; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.Interval; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; import com.beust.jcommander.Parameter; import com.google.inject.Injector; import dk.dma.ais.message.AisMessage; import dk.dma.ais.packet.AisPacket; import dk.dma.ais.packet.AisPacketFilters; import dk.dma.ais.packet.AisPacketOutputSinks; import dk.dma.ais.reader.AisReader; import dk.dma.ais.reader.AisReaders; import dk.dma.commons.app.AbstractCommandLineTool; import dk.dma.commons.util.io.OutputStreamSink; /** * @author Jens Tuxen * @author David Andersen Camre */ public class FileExportRest extends AbstractCommandLineTool { private FileOutputStream fileOutputStream; private OutputStream outputStream; private OutputStreamSink<AisPacket> sink; private AtomicLong counter; // Status Variables long timeStart = System.currentTimeMillis(); DecimalFormat decimalFormatter = new DecimalFormat("0.00"); // Meta data private long currentTimeStamp; private Long packageCount = 0L; private long lastFlushTimestamp; private long intervalStartTime; Interval intervalVar; private long lastLoadedTimestamp; private String metaFileName; String printPrefix = "[AIS-STORE] "; /** A date time formatter for utc. */ DateTimeFormatter dateTimeFormatter = DateTimeFormat.forPattern("yyyy-M-d'T'HH:mm:ss'Z"); @Parameter(names = "-mmsi", description = "Extract from mmsi schema") List<Integer> mmsis = new ArrayList<Integer>(); @Parameter(names = "-filter", description = "The filter to apply") String filter; @Parameter(names = "-interval", description = "The ISO 8601 time interval for data to export") String interval; @Parameter(names = { "-area", "-box", "-geo" }, description = "Extract from geopgraphic cells schema within bounding box lat1,lon1,lat2,lon2") String area; @Parameter(names = "-outputFormat", description = "Output format, options: raw, json, jsonObject (use -columns), kml, kmz, table (use -columns)") String outputFormat = "raw"; @Parameter(names = "-columns", description = "Optional columns, used for jsonObject and table.") String columns; @Parameter(names = "-separator", description = "Optional separator, used for table format.") String separator = ","; @Parameter(names = { "-file", "-output", "-o" }, description = "File to extract to (default is stdout)") String filePath; @Parameter(names = { "-fileName" }, description = "File to extract to (default is stdout)") String fileName; @Parameter(names = "-fetchSize", description = "internal fetch size buffer") Integer fetchSize = -1; @Parameter(names = "-force", description = "Disregard existing download and force redownload") boolean forceDownload = false; /** {@inheritDoc} */ @Override protected void run(Injector injector) throws Exception { printAisStoreNL("AIS STORE COMMAND LINE TOOL INITIATED"); printAisStoreLine(); // Hardcoded values // interval = "2015-1-5T14:00:00Z/2015-1-5T14:10:00Z"; // java -jar ais-store-cli-0.3-SNAPSHOT.jar export -area 15,-18,-10,14 -filter // "m.country=DNK & t.pos within bbox(15,-18,-10,14) & (t.lat<-0.3|t.lat>0.3) & (t.lon<-0.3|t.lon>0.3)" -fetchSize 30000 // -interval // java -jar ais-store-cli-0.3-SNAPSHOT.jar export -area 15,-18,-10,14 -filter // "m.country=DNK & t.pos within bbox(15,-18,-10,14) & (t.lat<-0.3|t.lat>0.3) & (t.lon<-0.3|t.lon>0.3)" -fetchSize 30000 // -interval 2015-1-5T14:00:00Z/2015-1-5T14:10:00Z // Create request String request = ""; if (interval == null || interval.equals("")) { printAisStoreNL("No Interval provided, please check your request."); // return; terminateAndPrintHelp(); } try { intervalVar = Interval.parse(interval); } catch (Exception e) { printAisStoreNL("Invalid Interval provided, please check your request."); terminateAndPrintHelp(); } // intervalVar = DateTimeUtil.toInterval(intervalStr); intervalStartTime = intervalVar.getStartMillis(); String intervalStr = interval.toString(); request = request + "?interval=" + interval; // System.out.println("Interval parsed correct " + intervalStr); // Check if interval is valid, throw exception etc // Create task for exception throwing if args are required // If error, throw exception of error description then -help if (mmsis.size() > 0) { request = request + "&mmsi="; for (int i = 0; i < mmsis.size() - 1; i++) { request = request + Integer.toString(mmsis.get(i)) + ","; } request = request + mmsis.get(mmsis.size() - 1); } // filter // Google URL Encoder // "t.name like H* & t.name like HAMLET" // CHeck if url filter is valid, then url encode it and add to request // filter = "t.name like H* & t.name like HAMLET"; // filter = "s.country in (DNK)"; if (filter != null && !filter.equals("")) { String encodedFilter = URLEncoder.encode(filter, "UTF-8"); try { AisPacketFilters.parseExpressionFilter(filter); } catch (Exception e) { printAisStoreNL("Invalid filter expression"); terminateAndPrintHelp(); } request = request + "&filter=" + encodedFilter; } // area // "&box=lat1,lon1,lat2,lon2" // blabla check if valid if (area != null && !area.equals("")) { request = request + "&box=" + area; } // If table, make sure column is added if ((columns == null || columns.equals("")) && (outputFormat.equals("table") || outputFormat.equals("jsonObject"))) { printAisStoreNL("When using outputFormat " + outputFormat + ", columns are required"); terminateAndPrintHelp(); } try { sink = AisPacketOutputSinks.getOutputSink(outputFormat, columns, separator); request = request + "&outputFormat=" + outputFormat; } catch (Exception e) { printAisStoreNL("Invalid output format provided, " + outputFormat + ", please check your request."); terminateAndPrintHelp(); } // if table do shit // columns \/ REQUIRED // "columns=mmsi;time;timestamp" // Check if valid // Split on ";" // "columns=<listElement0>:list1" if (columns != null) { request = request + "&columns=" + columns; } // seperator // "seperator=\t" // Url encode and add if (separator != null || !separator.equals("")) { String encodedSeparator = URLEncoder.encode(separator, "UTF-8"); request = request + "&seperator=" + encodedSeparator; } // fetchSize if (fetchSize != -1) { request = request + "&fetchsize=" + fetchSize; } // Get path from request, if none it will store in root of ais store client // filePath = "C:\\AisStoreData\\"; if (filePath == null) { filePath = ""; } else { filePath = filePath + "/"; } // No filename provided, generate unique based on request parameters if (fileName == null || fileName.equals("")) { MessageDigest md5 = MessageDigest.getInstance("MD5"); String hex = (new HexBinaryAdapter()).marshal(md5.digest(request.getBytes())); fileName = hex; } // Generate unique hashsum based on request metaFileName = fileName + ".aisstore"; // boolean isTryResume = true; // If we are trying to resume, don't override previous file try { fileOutputStream = new FileOutputStream(filePath + fileName, !forceDownload); } catch (Exception e) { printAisStoreNL("Error occuring writing to disk, make sure the folder path exists "); terminateAndPrintHelp(); } outputStream = new BufferedOutputStream(fileOutputStream); // Should we resume anything // We have read the file // If the file exists that means a previous transaction has been done // Do we resume, if we do, we need to find the resume point ie move the start interval /** * System.out.println("Test Compare"); System.out.println(intervalStr); * * DateTime time = new DateTime(interval.getStartMillis(), DateTimeZone.UTC); DateTime time2 = new * DateTime(interval.getEndMillis(), DateTimeZone.UTC); * * String newIntervalStr = dateTimeFormatter.withZoneUTC().print(time) + "/" + dateTimeFormatter.withZoneUTC().print(time2); * System.out.println(newIntervalStr); // DateTime dateTime = * dateTimeFormatter.parseDateTime("15-Oct-2013 11:34:26 AM").withZone(DateTimeZone.UTC); * * // System.out.println(dateTime); * * // Interval var = Interval.parse(intervalStr); // String dateStr = formatter.withZone(DateTimeZone.UTC).print(dateTime1); * // * * System.exit(0); **/ printAisStoreNL("Request generation complete."); printAisStoreNL("AIS Data will be saved to " + filePath + fileName); // System.out.println("--------------------------------------------------------------------------------"); // We are resuming, insert a Carriage Return Line Feed if (!forceDownload) { // Load the meta data in readMeta(); // We have processed some packages already if (packageCount != 0) { String str = "\r\n"; outputStream.write(str.getBytes()); printAisStoreLine(); printAisStoreNL("Resume detected - Updating Request"); // System.out.println("From " + intervalStr); // Update intervalStr DateTime time = new DateTime(lastLoadedTimestamp, DateTimeZone.UTC); DateTime time2 = new DateTime(intervalVar.getEndMillis(), DateTimeZone.UTC); intervalStr = dateTimeFormatter.withZoneUTC().print(time) + "/" + dateTimeFormatter.withZoneUTC().print(time2); // System.out.println("To " + intervalStr); printAisStoreLine(); // System.out.println("The last stored timestamp was \n" + lastLoadedTimestamp); // Interval interval2 = DateTimeUtil.toInterval(intervalStr); // System.out.println(interval2.getStartMillis()); } else { writeMetaInit(intervalVar.getStartMillis()); lastLoadedTimestamp = intervalVar.getStartMillis(); } } else { // We are starting a new request, create a new meta init writeMetaInit(intervalVar.getStartMillis()); lastLoadedTimestamp = intervalVar.getStartMillis(); } // System.out.println("Interval Str is " + intervalStr); // System.exit(0); // Initialize counter = new AtomicLong(packageCount); // Do we need to set a new interval start based on the meta data read? DefaultHttpClient httpClient = new DefaultHttpClient(); HttpHost target = new HttpHost("ais2.e-navigation.net", 443, "https"); request = "/aisview/rest/store/query" + request; HttpGet getRequest = new HttpGet(request); // HttpGet getRequest = new // HttpGet("/aisview/rest/store/query?interval=2015-1-1T10:00:00Z/2015-2-1T10:10:00Z&box=65.145,-5.373,34.450,76.893"); // + "&mmsi=219230000" // + "&mmsi=219230000" printAisStoreNL("Executing request to " + target); printAisStoreNL("Request is: " + request); HttpResponse httpResponse = httpClient.execute(target, getRequest); HttpEntity entity = httpResponse.getEntity(); // Check we have an OK from server etc. printAisStoreLine(); boolean terminateFailure = false; StatusLine reply = httpResponse.getStatusLine(); switch (reply.getStatusCode()) { case HttpStatus.SC_OK: printAisStoreNL("Server Accepted Connection, download will begin shortly"); printAisStoreLine(); break; default: printAisStoreNL("An error occured establishing connection to the server. "); printAisStoreNL( "Server returned Status Code " + reply.getStatusCode() + " with " + reply.getReasonPhrase()); terminateFailure = true; break; } if (terminateFailure) { return; } // System.out.println("Got reply " + reply.getReasonPhrase() + " status code " + reply.getStatusCode()); // String httpServerReply = httpResponse.getStatusLine(); // System.out.println(httpResponse.getStatusLine()); // // Header[] headers = httpResponse.getAllHeaders(); // for (int i = 0; i < headers.length; i++) { // System.out.println(headers[i]); // } // Do we use the footer? AisReader aisReader; if (entity != null) { InputStream inputStream = entity.getContent(); aisReader = aisReadWriter(inputStream); aisReader.start(); aisReader.join(); // Write the remainder still stored in buffer, update the final meta data with the finished data writeMetaUpdate(currentTimeStamp, counter.get()); // Write the footer sink.footer(outputStream, counter.get()); // Closer and flush the buffer outputStream.flush(); outputStream.close(); // Close and flush the file stream fileOutputStream.flush(); fileOutputStream.close(); } // print a new line to move on from previous /r printAisStoreNL( "Downloading AIS Data 100% Estimated Time Left: 00:00:00 "); printAisStoreLine(); printAisStoreNL("DOWNLOAD SUCCESS"); printAisStoreLine(); // We know current time long currentTime = System.currentTimeMillis(); // How long have we been running long millis = currentTime - timeStart; String timeLeftStr = String.format("%02d:%02d:%02d", TimeUnit.MILLISECONDS.toHours(millis), TimeUnit.MILLISECONDS.toMinutes(millis) - TimeUnit.HOURS.toMinutes(TimeUnit.MILLISECONDS.toHours(millis)), TimeUnit.MILLISECONDS.toSeconds(millis) - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(millis))); printAisStoreNL("Total Time " + timeLeftStr); printAisStoreNL("Finished at: " + new Date()); printAisStoreNL("Messages recieved " + counter); // printAisStore("Query took " + timeLeftStr); } private void printDownloadStatus() { // Determine how far left we have to go // We know current time long currentTime = System.currentTimeMillis(); // How long have we been running long milisecondsRunning = currentTime - timeStart; // Calculate total % done: // Total AIS time to process in miliseconds long aisTimeTotalOriginal = intervalVar.getEndMillis() - intervalStartTime; // Stretch of processed AIS time: long processedAisTimeOriginal = lastFlushTimestamp - intervalStartTime; double percentDoneOriginal = (double) processedAisTimeOriginal / (double) aisTimeTotalOriginal * 100; // Calculate the estimated time // Total AIS time to process in miliseconds long aisTimeTotal = intervalVar.getEndMillis() - lastLoadedTimestamp; // Stretch of processed AIS time: long processedAisTime = lastFlushTimestamp - lastLoadedTimestamp; double percentDoneNow = (double) processedAisTime / (double) aisTimeTotal * 100; double goToPercent = (100 - percentDoneNow); // How many % do we calculate pr. milisecond double percentPrMilisecond = percentDoneNow / ((double) milisecondsRunning); double timeLeft = goToPercent / percentPrMilisecond; long millis = (long) timeLeft; String timeLeftStr = String.format("%02d:%02d:%02d", TimeUnit.MILLISECONDS.toHours(millis), TimeUnit.MILLISECONDS.toMinutes(millis) - TimeUnit.HOURS.toMinutes(TimeUnit.MILLISECONDS.toHours(millis)), TimeUnit.MILLISECONDS.toSeconds(millis) - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(millis))); String percentDoneStr = decimalFormatter.format(percentDoneOriginal) + "%"; // String percentDoneStr = ((double) ((int) (percentDoneOriginal * 100))) / 100 + "%"; // String part1DownloadMessage = "Downloading AIS Data " + percentDoneStr; // String part2DownloadMessage = " Estimated Time Left: " + timeLeftStr; // if (part1DownloadMessage.length() < ) printAisStore("Downloading AIS Data " + percentDoneStr + " Estimated Time Left: " + timeLeftStr + "\r"); // printAisStore(); // int seconds = (int) (timeLeft / 1000) % 60; // int minutes = (int) ((timeLeft / (1000 * 60)) % 60); // int hours = (int) ((timeLeft / (1000 * 60 * 60)) % 24); // System.out.println(hours + ":" + minutes + ":" + seconds); // System.out.println("Miliseconds running" + milisecondsRunning); // // // AIS Time left of request // long aisTimeLeft = intervalVar.getEndMillis() - lastFlushTimestamp; // // // How long have we spent parsing up to now // long aisTimeParsed = processedAisTime / milisecondsRunning; // // System.out.println("We have in " + milisecondsRunning + " miliseconds processed " + processedAisTime + " AIS Interval"); // System.out.println("AIS Time Left: " + aisTimeLeft); // System.out.println("aisTimeParsed " + aisTimeParsed); // counterCurrent.get(); // // If we are resuming our % will be further along // We know our count since start // We know how long we have been running } /** * Read the meta file containing data regarding previous transactions in * * The meta file will have the following fields: file name - the file containing the data, if no filename is provided in request * we use hash of request to ensure uniqueness last time stamp - the last recorded timestamp packageCount - the amount of * messages written, used in footer * * @param path * @param metaFileName */ private void readMeta() throws IOException, ParseException { File f = new File(filePath + metaFileName); if (f.exists() && !f.isDirectory()) { JSONParser parser = new JSONParser(); try { Object obj = parser.parse(new FileReader(filePath + metaFileName)); JSONObject jsonObject = (JSONObject) obj; // if (jsonObject.get("filename") != null) { fileName = (String) jsonObject.get("filename"); // System.out.println(fileName); // } lastLoadedTimestamp = (Long) jsonObject.get("timestamp"); // System.out.println(lastFlushTimestamp); packageCount = (Long) jsonObject.get("packageCount"); // System.out.println(packageCount); } catch (Exception e) { e.printStackTrace(); System.out.println("No meta file detected or invalid meta file"); } } } @SuppressWarnings("unchecked") private void writeMetaInit(Long startInterval) { JSONObject obj = new JSONObject(); obj.put("filename", fileName); obj.put("timestamp", startInterval); obj.put("packageCount", packageCount); try { // System.out.println("Writing"); FileWriter file = new FileWriter(filePath + metaFileName); file.write(obj.toJSONString()); file.flush(); file.close(); } catch (IOException e) { e.printStackTrace(); } } @SuppressWarnings("unchecked") private void writeMetaUpdate(long timestamp, long packageCount) { // System.out.println("Updating meta!"); FileWriter file = null; JSONObject obj = new JSONObject(); obj.put("filename", fileName); obj.put("timestamp", timestamp); obj.put("packageCount", new Long(packageCount)); try { file = new FileWriter(filePath + metaFileName); file.write(obj.toJSONString()); file.flush(); } catch (IOException e) { // e.printStackTrace(); System.out.println("Failed to write because reasons " + e.getMessage()); } finally { try { file.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); System.out.println("Failed to close file! " + e.getMessage()); } } } private AisReader aisReadWriter(InputStream in) throws Exception { AisReader r = AisReaders.createReaderFromInputStream(in); r.registerPacketHandler(new Consumer<AisPacket>() { @Override public void accept(AisPacket t) { AisMessage message = t.tryGetAisMessage(); // System.out.println(message.toString()); currentTimeStamp = t.getBestTimestamp(); if (lastLoadedTimestamp >= currentTimeStamp) { // System.out.println("Skipping Message!"); return; } if (message == null) { return; } try { sink.process(outputStream, t, counter.incrementAndGet()); if (currentTimeStamp != lastFlushTimestamp && counter.get() % 10000 == 0) { // We have a new timestamp sequence lastFlushTimestamp = currentTimeStamp; writeMetaUpdate(lastFlushTimestamp, counter.get()); // Force flush on both outputStream.flush(); fileOutputStream.flush(); // Write // lastFlushTimestamp // timestamp // if (counter.get() >= 1000) { // System.out.println("Terminating as part of test! Last written timestamp was " + lastFlushTimestamp); // System.exit(0); // } // Update user on progress printDownloadStatus(); } // } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } }); return r; } public static void main(String[] args) throws Exception { new FileExportRest().execute(args); } private void terminateAndPrintHelp() { System.out.println("Terminating AIS Store Command Line"); try { new FileExportRest().execute(new String[] { "-help" }); } catch (Exception e) { } System.exit(-1); } private void printAisStore(String toPrint) { System.out.print(printPrefix + toPrint); } private void printAisStoreNL(String toPrint) { System.out.println(printPrefix + toPrint); } private void printAisStoreLine() { System.out.println( printPrefix + "--------------------------------------------------------------------------------"); } }