List of usage examples for java.time.format DateTimeFormatter ISO_DATE
DateTimeFormatter ISO_DATE
To view the source code for java.time.format DateTimeFormatter ISO_DATE.
Click Source Link
From source file:lumbermill.internal.elasticsearch.ElasticSearchOkHttpClientImpl.java
private String indexRowWithDateAndType(JsonEvent event) { Optional<String> formattedType = type.format(event); if (!formattedType.isPresent()) { throw new IllegalStateException( "Issue with type, could not extract field from event " + type.original()); }/*from w w w .jav a 2 s . c o m*/ Optional<String> formattedIndex = index.format(event); if (!formattedIndex.isPresent()) { throw new IllegalStateException( "Issue with index, could not extract field from event: " + index.original()); } Optional<String> formattedDocumentId = Optional.empty(); if (documentId.isPresent()) { formattedDocumentId = documentId.get().format(event); if (!formattedDocumentId.isPresent()) { throw new IllegalStateException( "Issue with index, could not extract field from event: " + index.original()); } } ObjectNode objectNode = OBJECT_MAPPER.createObjectNode(); ObjectNode data = OBJECT_MAPPER.createObjectNode(); // Prepare for adding day to index for each event if (indexIsPrefix) { LocalDate indexDate; // TODO: Not sure how to handle this... what should be the behaviour if the specified timestamp field // does not exist if (event.has(this.timestampField)) { indexDate = LocalDate.parse(event.valueAsString(this.timestampField).substring(0, 10), DateTimeFormatter.ISO_DATE); } else { indexDate = LocalDate.now(); } data.put("_index", formattedIndex.get() + indexDate.format(DateTimeFormatter.ofPattern("yyyy.MM.dd"))); } else { data.put("_index", formattedIndex.get()); } data.put("_type", formattedType.get()); if (formattedDocumentId.isPresent()) { data.put("_id", formattedDocumentId.get()); } objectNode.set("index", data); return objectNode.toString(); }
From source file:com.mycompany.trafficimportfileconverter2.Main2Controller.java
private void loadInInfo(File file) { BufferedReader TSVFile = null; //get rid of any dates and data already there. dates.clear();/*from w w w . j a v a2 s . c o m*/ data.clear(); try { TSVFile = new BufferedReader(new FileReader(file.getAbsoluteFile())); String dataRow = TSVFile.readLine(); while (dataRow != null) { String[] row = dataRow.split("\t"); System.out.println(row[1] + "Will be:\n" + row[2]); dates.add(LocalDate.parse("20" + row[0])); data.add(getData(row)); dataRow = TSVFile.readLine(); } TSVFile.close(); log("Days in file: " + dates.size()); if (dates.size() > datePickers.length) { log("WARNING: More days found in file than can be displayed!\n" + "Only the first " + datePickers.length + " " + "can be displayed and configurable.\nThe last " + (dates.size() - datePickers.length) + " " + "dates in your file will be\nimported with" + " the dates untouched!\n\n"); log("Unconfigurable days that will be imported:"); for (int i = datePickers.length; i < dates.size(); i++) { log(dates.get(i).format(DateTimeFormatter.ISO_DATE)); } } System.out.println(); } catch (FileNotFoundException ex) { Logger.getLogger(Main2Controller.class.getName()).log(Level.SEVERE, null, ex); } catch (IOException ex) { Logger.getLogger(Main2Controller.class.getName()).log(Level.SEVERE, null, ex); } catch (ArrayIndexOutOfBoundsException e) { Logger.getLogger(Main2Controller.class.getName()).log(Level.SEVERE, null, e); String str = "It appears like the file contents were invalid.\n" + "Please try again or a different file."; System.out.println(str); log(str); } finally { try { TSVFile.close(); } catch (IOException ex) { Logger.getLogger(Main2Controller.class.getName()).log(Level.SEVERE, null, ex); } } }
From source file:investiagenofx2.view.InvestiaGenOFXController.java
private void getAccountsFromWeb() { String name = "Nom Pastrouv"; try {/* w w w . ja v a2 s . c o m*/ name = ((HtmlDivision) htmlPage.getHtmlElementById("phase1CustomerName")).asText(); name = name.replace("Mme ", "").trim(); name = name.replace("M. ", "").trim(); } catch (ElementNotFoundException ex) { try { name = ((HtmlSelect) htmlPage.getHtmlElementById("selAccount")).asText(); name = name.replaceAll("[^A-Za-z ]", "").trim(); } catch (ElementNotFoundException ex2) { } } name = capitalizeFully(name); String[] token = name.split(" "); accountOwnerName = token[0] + " " + token[1]; token = ((HtmlDivision) htmlPage.getByXPath("//div[contains(@class, 'col-xs-4 text-right')]").get(0)) .asText().split(" "); dataAsDate = LocalDate.parse(token[token.length - 1], DateTimeFormatter.ISO_DATE); accounts = new ArrayList<>(); try { if (InvestiaGenOFX.debug) { htmlPage = InvestiaGenOFX.getWebClient().getPage(InvestiaGenOFX.debugFullPath + "-Investments.htm"); } else { htmlPage = InvestiaGenOFX.getWebClient() .getPage(txt_investiaURL.getText() + "/Investments/ValueOf" + "?wcag=true"); } List<?> sections = htmlPage.getByXPath("//section[not(contains(@id, 'valueOf'))]"); for (int i = 1; i < sections.size(); i++) { HtmlSection section = (HtmlSection) sections.get(i); HtmlHeading4 h4 = (HtmlHeading4) section.getElementsByTagName("h4").get(0); token = h4.asText().replaceAll("[\\\\,]", "").split(" - "); Account account = new Account(token[0], token[1], dataAsDate); try { HtmlHeading5 h5 = (HtmlHeading5) section.getElementsByTagName("h5").get(0); token = h5.asText().split(" "); if (token[0].startsWith("Numro")) { account.setAccountID(token[token.length - 1]); } } catch (IndexOutOfBoundsException ex) { } addAccount(account); } } catch (Exception ex) { Logger.getLogger(InvestiaGenOFXController.class.getName()).log(Level.SEVERE, null, ex); } }
From source file:com.mycompany.trafficimportfileconverter2.Main2Controller.java
/** * * @param file The location of the file downloaded from the google sheet. * @return The dates the scheduled traffic. *//*from ww w .java2 s . com*/ // private ArrayList<LocalDate> parseDates(File file) { // BufferedReader TSVFile = null; // ArrayList<LocalDate> answer = new ArrayList<>(7); // try { // TSVFile = new BufferedReader(new FileReader(file.getAbsoluteFile())); // String dataRow = TSVFile.readLine(); // while (dataRow != null) { // String[] row = dataRow.split("\t"); // System.out.println("I think the day is " + row[0]); // answer.add(LocalDate.parse("20" + row[0])); // dataRow = TSVFile.readLine(); // } // TSVFile.close(); //// return answer; // } catch (FileNotFoundException ex) { // Logger.getLogger(Main2Controller.class.getName()).log(Level.SEVERE, null, ex); // } catch (IOException ex) { // Logger.getLogger(Main2Controller.class.getName()).log(Level.SEVERE, null, ex); // } finally { // try { // TSVFile.close(); // } catch (IOException ex) { // Logger.getLogger(Main2Controller.class.getName()).log(Level.SEVERE, null, ex); // } // // } // return answer; // } private void handleFileWriting() { //ArrayList<String> filenames = new ArrayList<>(dates.size()); dates.parallelStream().forEach(x -> { int index = dates.indexOf(x); if (!dateChecks[index].isSelected()) { return; } //Am i already imported? File alreadyThereFile = isPresentForDate(x); if (null != alreadyThereFile) { UI(() -> { Alert alert = new Alert(Alert.AlertType.CONFIRMATION); alert.setTitle("Re-import?"); alert.setContentText("Hey there, just thought I'd let you know " + "that you've already imported a file for this day: " + x.format(DateTimeFormatter.ISO_DATE) + ". Because I found this file: " + alreadyThereFile.getName() + "\nAre you sure you want to do this?"); Optional<ButtonType> answer = alert.showAndWait(); if (answer.isPresent() && answer.get().getButtonData().equals(ButtonData.OK_DONE)) { writeFile(toWideOrbitTitle(x), data.get(index)); UI(() -> { datePickers[index].setStyle("-fx-background-color: yellow"); dayLabels[index].setStyle("-fx-background-color: yellow"); }); synchronized (lock) { myfiles.add(toWideOrbitTitle(x)); System.out.println("Adding: " + toWideOrbitTitle(x)); System.out.println(myfiles.toString()); } } else { log("Skipping: " + alreadyThereFile.getName()); } }); } else { writeFile(toWideOrbitTitle(x), data.get(index)); UI(() -> { datePickers[index].setStyle("-fx-background-color: yellow"); dayLabels[index].setStyle("-fx-background-color: yellow"); }); synchronized (lock) { myfiles.add(toWideOrbitTitle(x)); } } }); }
From source file:net.tradelib.apps.StrategyBacktest.java
public static void run(Strategy strategy) throws Exception { // Setup the logging System.setProperty("java.util.logging.SimpleFormatter.format", "%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS: %4$s: %5$s%n%6$s%n"); LogManager.getLogManager().reset(); Logger rootLogger = Logger.getLogger(""); if (Boolean.parseBoolean(BacktestCfg.instance().getProperty("file.log", "true"))) { FileHandler logHandler = new FileHandler("diag.out", 8 * 1024 * 1024, 2, true); logHandler.setFormatter(new SimpleFormatter()); logHandler.setLevel(Level.FINEST); rootLogger.addHandler(logHandler); }//w ww . j a va2 s. co m if (Boolean.parseBoolean(BacktestCfg.instance().getProperty("console.log", "true"))) { ConsoleHandler consoleHandler = new ConsoleHandler(); consoleHandler.setFormatter(new SimpleFormatter()); consoleHandler.setLevel(Level.INFO); rootLogger.addHandler(consoleHandler); } rootLogger.setLevel(Level.INFO); // Setup Hibernate // Configuration configuration = new Configuration(); // StandardServiceRegistryBuilder builder = new StandardServiceRegistryBuilder().applySettings(configuration.getProperties()); // SessionFactory factory = configuration.buildSessionFactory(builder.build()); Context context = new Context(); context.dbUrl = BacktestCfg.instance().getProperty("db.url"); HistoricalDataFeed hdf = new SQLDataFeed(context); hdf.configure(BacktestCfg.instance().getProperty("datafeed.config", "config/datafeed.properties")); context.historicalDataFeed = hdf; HistoricalReplay hr = new HistoricalReplay(context); context.broker = hr; strategy.initialize(context); strategy.cleanupDb(); long start = System.nanoTime(); strategy.start(); long elapsedTime = System.nanoTime() - start; System.out.println("backtest took " + String.format("%.2f secs", (double) elapsedTime / 1e9)); start = System.nanoTime(); strategy.updateEndEquity(); strategy.writeExecutionsAndTrades(); strategy.writeEquity(); elapsedTime = System.nanoTime() - start; System.out .println("writing to the database took " + String.format("%.2f secs", (double) elapsedTime / 1e9)); System.out.println(); // Write the strategy totals to the database strategy.totalTradeStats(); // Write the strategy report to the database and obtain the JSON // for writing it to the console. JsonObject report = strategy.writeStrategyReport(); JsonArray asa = report.getAsJsonArray("annual_stats"); String csvPath = BacktestCfg.instance().getProperty("positions.csv.prefix"); if (!Strings.isNullOrEmpty(csvPath)) { csvPath += "-" + strategy.getLastTimestamp().toLocalDate().format(DateTimeFormatter.BASIC_ISO_DATE) + ".csv"; } String ordersCsvPath = BacktestCfg.instance().getProperty("orders.csv.suffix"); if (!Strings.isNullOrEmpty(ordersCsvPath)) { ordersCsvPath = strategy.getLastTimestamp().toLocalDate().format(DateTimeFormatter.BASIC_ISO_DATE) + "-" + strategy.getName() + ordersCsvPath; } String actionsPath = BacktestCfg.instance().getProperty("actions.file.suffix"); if (!Strings.isNullOrEmpty(actionsPath)) { actionsPath = strategy.getLastTimestamp().toLocalDate().format(DateTimeFormatter.BASIC_ISO_DATE) + "-" + strategy.getName() + actionsPath; } // If emails are being send out String signalText = StrategyText.build(context.dbUrl, strategy.getName(), strategy.getLastTimestamp().toLocalDate(), " ", csvPath, '|'); System.out.println(signalText); System.out.println(); if (!Strings.isNullOrEmpty(ordersCsvPath)) { StrategyText.buildOrdersCsv(context.dbUrl, strategy.getName(), strategy.getLastTimestamp().toLocalDate(), ordersCsvPath); } File actionsFile = Strings.isNullOrEmpty(actionsPath) ? null : new File(actionsPath); if (actionsFile != null) { FileUtils.writeStringToFile(actionsFile, signalText + System.getProperty("line.separator") + System.getProperty("line.separator")); } String message = ""; if (asa.size() > 0) { // Sort the array TreeMap<Integer, Integer> map = new TreeMap<Integer, Integer>(); for (int ii = 0; ii < asa.size(); ++ii) { int year = asa.get(ii).getAsJsonObject().get("year").getAsInt(); map.put(year, ii); } for (int id : map.values()) { JsonObject jo = asa.get(id).getAsJsonObject(); String yearStr = String.valueOf(jo.get("year").getAsInt()); String pnlStr = String.format("$%,d", jo.get("pnl").getAsInt()); String pnlPctStr = String.format("%.2f%%", jo.get("pnl_pct").getAsDouble()); String endEqStr = String.format("$%,d", jo.get("end_equity").getAsInt()); String ddStr = String.format("$%,d", jo.get("maxdd").getAsInt()); String ddPctStr = String.format("%.2f%%", jo.get("maxdd_pct").getAsDouble()); String str = yearStr + " PnL: " + pnlStr + ", PnL Pct: " + pnlPctStr + ", End Equity: " + endEqStr + ", MaxDD: " + ddStr + ", Pct MaxDD: " + ddPctStr; message += str + "\n"; } String pnlStr = String.format("$%,d", report.get("pnl").getAsInt()); String pnlPctStr = String.format("%.2f%%", report.get("pnl_pct").getAsDouble()); String ddStr = String.format("$%,d", report.get("avgdd").getAsInt()); String ddPctStr = String.format("%.2f%%", report.get("avgdd_pct").getAsDouble()); String gainToPainStr = String.format("%.4f", report.get("gain_to_pain").getAsDouble()); String str = "\nAvg PnL: " + pnlStr + ", Pct Avg PnL: " + pnlPctStr + ", Avg DD: " + ddStr + ", Pct Avg DD: " + ddPctStr + ", Gain to Pain: " + gainToPainStr; message += str + "\n"; } else { message += "\n"; } // Global statistics JsonObject jo = report.getAsJsonObject("total_peak"); String dateStr = jo.get("date").getAsString(); int maxEndEq = jo.get("equity").getAsInt(); jo = report.getAsJsonObject("total_maxdd"); double cash = jo.get("cash").getAsDouble(); double pct = jo.get("pct").getAsDouble(); message += "\n" + "Total equity peak [" + dateStr + "]: " + String.format("$%,d", maxEndEq) + "\n" + String.format("Current Drawdown: $%,d [%.2f%%]", Math.round(cash), pct) + "\n"; if (report.has("latest_peak") && report.has("latest_maxdd")) { jo = report.getAsJsonObject("latest_peak"); LocalDate ld = LocalDate.parse(jo.get("date").getAsString(), DateTimeFormatter.ISO_DATE); maxEndEq = jo.get("equity").getAsInt(); jo = report.getAsJsonObject("latest_maxdd"); cash = jo.get("cash").getAsDouble(); pct = jo.get("pct").getAsDouble(); message += "\n" + Integer.toString(ld.getYear()) + " equity peak [" + ld.format(DateTimeFormatter.ISO_DATE) + "]: " + String.format("$%,d", maxEndEq) + "\n" + String.format("Current Drawdown: $%,d [%.2f%%]", Math.round(cash), pct) + "\n"; } message += "\n" + "Avg Trade PnL: " + String.format("$%,d", Math.round(report.get("avg_trade_pnl").getAsDouble())) + ", Max DD: " + String.format("$%,d", Math.round(report.get("maxdd").getAsDouble())) + ", Max DD Pct: " + String.format("%.2f%%", report.get("maxdd_pct").getAsDouble()) + ", Num Trades: " + Integer.toString(report.get("num_trades").getAsInt()); System.out.println(message); if (actionsFile != null) { FileUtils.writeStringToFile(actionsFile, message + System.getProperty("line.separator"), true); } if (Boolean.parseBoolean(BacktestCfg.instance().getProperty("email.enabled", "false"))) { Properties props = new Properties(); props.put("mail.smtp.auth", "true"); props.put("mail.smtp.starttls.enable", "true"); props.put("mail.smtp.host", "smtp.sendgrid.net"); props.put("mail.smtp.port", "587"); String user = BacktestCfg.instance().getProperty("email.user"); String pass = BacktestCfg.instance().getProperty("email.pass"); Session session = Session.getInstance(props, new javax.mail.Authenticator() { protected PasswordAuthentication getPasswordAuthentication() { return new PasswordAuthentication(user, pass); } }); MimeMessage msg = new MimeMessage(session); try { msg.setFrom(new InternetAddress(BacktestCfg.instance().getProperty("email.from"))); msg.addRecipients(RecipientType.TO, BacktestCfg.instance().getProperty("email.recipients")); msg.setSubject(strategy.getName() + " Report [" + strategy.getLastTimestamp().format(DateTimeFormatter.ISO_LOCAL_DATE) + "]"); msg.setText("Positions & Signals\n" + signalText + "\n\nStatistics\n" + message); Transport.send(msg); } catch (Exception ee) { Logger.getLogger("").warning(ee.getMessage()); } } if (Boolean.parseBoolean(BacktestCfg.instance().getProperty("sftp.enabled", "false"))) { HashMap<String, String> fileMap = new HashMap<String, String>(); if (!Strings.isNullOrEmpty(actionsPath)) fileMap.put(actionsPath, actionsPath); if (!Strings.isNullOrEmpty(ordersCsvPath)) fileMap.put(ordersCsvPath, ordersCsvPath); String user = BacktestCfg.instance().getProperty("sftp.user"); String pass = BacktestCfg.instance().getProperty("sftp.pass"); String host = BacktestCfg.instance().getProperty("sftp.host"); SftpUploader sftp = new SftpUploader(host, user, pass); sftp.upload(fileMap); } }
From source file:net.tradelib.core.Series.java
static public Series fromCsv(String path, boolean header, DateTimeFormatter dtf, LocalTime lt) throws Exception { if (dtf == null) { if (lt == null) dtf = DateTimeFormatter.ISO_LOCAL_DATE_TIME; else/* ww w .j a va 2 s . c o m*/ dtf = DateTimeFormatter.ISO_DATE; } // Parse and import the csv CSVFormat csvFmt = CSVFormat.DEFAULT.withCommentMarker('#').withIgnoreSurroundingSpaces(); if (header) csvFmt = csvFmt.withHeader(); CSVParser csv = csvFmt.parse(new BufferedReader(new FileReader(path))); int ncols = -1; Series result = null; double[] values = null; for (CSVRecord rec : csv.getRecords()) { if (result == null) { ncols = rec.size() - 1; values = new double[ncols]; result = new Series(ncols); } for (int ii = 0; ii < ncols; ++ii) { values[ii] = Double.parseDouble(rec.get(ii + 1)); } LocalDateTime ldt; if (lt != null) { ldt = LocalDate.parse(rec.get(0), dtf).atTime(lt); } else { ldt = LocalDateTime.parse(rec.get(0), dtf); } result.append(ldt, values); } if (header) { Map<String, Integer> headerMap = csv.getHeaderMap(); result.clearNames(); for (Map.Entry<String, Integer> me : headerMap.entrySet()) { if (me.getValue() > 0) result.setName(me.getKey(), me.getValue() - 1); } } return result; }
From source file:org.openlmis.converter.DirectDateTypeConverter.java
private String parseDate(String value) { LocalDate date = null;// w w w . jav a2 s . c o m for (String format : DATE_FORMATS) { try { date = LocalDate.parse(value, DateTimeFormatter.ofPattern(format)); break; } catch (DateTimeParseException exp) { date = null; logger.debug("Can't parse date {} with format {}", value, format, exp); } } return null == date ? null : date.format(DateTimeFormatter.ISO_DATE); }
From source file:org.talend.dataquality.statistics.datetime.utils.PatternListGenerator.java
@SuppressWarnings("unused") private static void validateISOPattens(List<String> isoPatternList) { Set<String> formattedDateTimeSet = new HashSet<String>(); for (String pattern : isoPatternList) { formattedDateTimeSet.add(getFormattedDateTime(pattern, Locale.US)); }//from w w w. j av a2 s . co m DateTimeFormatter[] formatters = new DateTimeFormatter[] { DateTimeFormatter.BASIC_ISO_DATE, // 1 DateTimeFormatter.ISO_DATE, // 2 DateTimeFormatter.ISO_DATE_TIME, // 3 // DateTimeFormatter.ISO_TIME, // DateTimeFormatter.ISO_INSTANT, // 4 DateTimeFormatter.ISO_LOCAL_DATE, // 5 DateTimeFormatter.ISO_LOCAL_DATE_TIME, // 6 // DateTimeFormatter.ISO_LOCAL_TIME, // DateTimeFormatter.ISO_OFFSET_DATE, // 7 DateTimeFormatter.ISO_OFFSET_DATE_TIME, // 8 // DateTimeFormatter.ISO_OFFSET_TIME, // DateTimeFormatter.ISO_ORDINAL_DATE, // 9 DateTimeFormatter.ISO_WEEK_DATE, // 10 DateTimeFormatter.ISO_ZONED_DATE_TIME, // 11 DateTimeFormatter.RFC_1123_DATE_TIME, // 12 }; System.out.println("-------------Validate ISO PattenText-------------"); for (int i = 0; i < formatters.length; i++) { System.out.print((i + 1) + "\t"); try { String formattedDateTime = ZONED_DATE_TIME.format(formatters[i]); System.out.print(formattedDateTimeSet.contains(formattedDateTime) ? "YES\t" : "NO\t"); System.out.println(formattedDateTime); } catch (Throwable t) { System.out.println(t.getMessage()); } } }
From source file:ro.cs.products.Executor.java
private static int execute(CommandLine commandLine) throws Exception { int retCode = ReturnCode.OK; CommandLineParser parser = new DefaultParser(); String logFile = props.getProperty("master.log.file"); String folder;//from w w w .j a v a 2 s . c om boolean debugMode = commandLine.hasOption(Constants.PARAM_VERBOSE); Logger.CustomLogger logger; SensorType sensorType = commandLine.hasOption(Constants.SENSOR) ? Enum.valueOf(SensorType.class, commandLine.getOptionValue(Constants.SENSOR)) : SensorType.S2; if (commandLine.hasOption(Constants.PARAM_INPUT_FOLDER)) { folder = commandLine.getOptionValue(Constants.PARAM_INPUT_FOLDER); Utilities.ensureExists(Paths.get(folder)); Logger.initialize(Paths.get(folder, logFile).toAbsolutePath().toString(), debugMode); logger = Logger.getRootLogger(); if (commandLine.hasOption(Constants.PARAM_VERBOSE)) { printCommandLine(commandLine); } if (sensorType == SensorType.L8) { logger.warn("Argument --input will be ignored for Landsat8"); } else { String rootFolder = commandLine.getOptionValue(Constants.PARAM_INPUT_FOLDER); FillAnglesMethod fillAnglesMethod = Enum.valueOf(FillAnglesMethod.class, commandLine.hasOption(Constants.PARAM_FILL_ANGLES) ? commandLine.getOptionValue(Constants.PARAM_FILL_ANGLES).toUpperCase() : FillAnglesMethod.NONE.name()); if (!FillAnglesMethod.NONE.equals(fillAnglesMethod)) { try { Set<String> products = null; if (commandLine.hasOption(Constants.PARAM_PRODUCT_LIST)) { products = new HashSet<>(); for (String product : commandLine.getOptionValues(Constants.PARAM_PRODUCT_LIST)) { if (!product.endsWith(".SAFE")) { products.add(product + ".SAFE"); } else { products.add(product); } } } ProductInspector inspector = new ProductInspector(rootFolder, fillAnglesMethod, products); inspector.traverse(); } catch (IOException e) { logger.error(e.getMessage()); retCode = ReturnCode.DOWNLOAD_ERROR; } } } } else { folder = commandLine.getOptionValue(Constants.PARAM_OUT_FOLDER); Utilities.ensureExists(Paths.get(folder)); Logger.initialize(Paths.get(folder, logFile).toAbsolutePath().toString(), debugMode); logger = Logger.getRootLogger(); printCommandLine(commandLine); String proxyType = commandLine.hasOption(Constants.PARAM_PROXY_TYPE) ? commandLine.getOptionValue(Constants.PARAM_PROXY_TYPE) : nullIfEmpty(props.getProperty("proxy.type", null)); String proxyHost = commandLine.hasOption(Constants.PARAM_PROXY_HOST) ? commandLine.getOptionValue(Constants.PARAM_PROXY_HOST) : nullIfEmpty(props.getProperty("proxy.host", null)); String proxyPort = commandLine.hasOption(Constants.PARAM_PROXY_PORT) ? commandLine.getOptionValue(Constants.PARAM_PROXY_PORT) : nullIfEmpty(props.getProperty("proxy.port", null)); String proxyUser = commandLine.hasOption(Constants.PARAM_PROXY_USER) ? commandLine.getOptionValue(Constants.PARAM_PROXY_USER) : nullIfEmpty(props.getProperty("proxy.user", null)); String proxyPwd = commandLine.hasOption(Constants.PARAM_PROXY_PASSWORD) ? commandLine.getOptionValue(Constants.PARAM_PROXY_PASSWORD) : nullIfEmpty(props.getProperty("proxy.pwd", null)); NetUtils.setProxy(proxyType, proxyHost, proxyPort == null ? 0 : Integer.parseInt(proxyPort), proxyUser, proxyPwd); List<ProductDescriptor> products = new ArrayList<>(); Set<String> tiles = new HashSet<>(); Polygon2D areaOfInterest = new Polygon2D(); ProductStore source = Enum.valueOf(ProductStore.class, commandLine.getOptionValue(Constants.PARAM_DOWNLOAD_STORE, ProductStore.SCIHUB.toString())); if (sensorType == SensorType.S2 && !commandLine.hasOption(Constants.PARAM_FLAG_SEARCH_AWS) && !commandLine.hasOption(Constants.PARAM_USER)) { throw new MissingOptionException("Missing SciHub credentials"); } String user = commandLine.getOptionValue(Constants.PARAM_USER); String pwd = commandLine.getOptionValue(Constants.PARAM_PASSWORD); if (user != null && pwd != null && !user.isEmpty() && !pwd.isEmpty()) { String authToken = "Basic " + new String(Base64.getEncoder().encode((user + ":" + pwd).getBytes())); NetUtils.setAuthToken(authToken); } ProductDownloader downloader = sensorType.equals(SensorType.S2) ? new SentinelProductDownloader(source, commandLine.getOptionValue(Constants.PARAM_OUT_FOLDER), props) : new LandsatProductDownloader(commandLine.getOptionValue(Constants.PARAM_OUT_FOLDER), props); TileMap tileMap = sensorType == SensorType.S2 ? SentinelTilesMap.getInstance() : LandsatTilesMap.getInstance(); if (commandLine.hasOption(Constants.PARAM_AREA)) { String[] points = commandLine.getOptionValues(Constants.PARAM_AREA); for (String point : points) { areaOfInterest.append(Double.parseDouble(point.substring(0, point.indexOf(","))), Double.parseDouble(point.substring(point.indexOf(",") + 1))); } } else if (commandLine.hasOption(Constants.PARAM_AREA_FILE)) { areaOfInterest = Polygon2D.fromWKT(new String( Files.readAllBytes(Paths.get(commandLine.getOptionValue(Constants.PARAM_AREA_FILE))), StandardCharsets.UTF_8)); } else if (commandLine.hasOption(Constants.PARAM_TILE_SHAPE_FILE)) { String tileShapeFile = commandLine.getOptionValue(Constants.PARAM_TILE_SHAPE_FILE); if (Files.exists(Paths.get(tileShapeFile))) { logger.info(String.format("Reading %s tiles extents", sensorType)); tileMap.fromKmlFile(tileShapeFile); logger.info(String.valueOf(tileMap.getCount() + " tiles found")); } } else { if (tileMap.getCount() == 0) { logger.info(String.format("Loading %s tiles extents", sensorType)); tileMap.read(Executor.class.getResourceAsStream(sensorType + "tilemap.dat")); logger.info(String.valueOf(tileMap.getCount() + " tile extents loaded")); } } if (commandLine.hasOption(Constants.PARAM_TILE_LIST)) { Collections.addAll(tiles, commandLine.getOptionValues(Constants.PARAM_TILE_LIST)); } else if (commandLine.hasOption(Constants.PARAM_TILE_LIST_FILE)) { tiles.addAll( Files.readAllLines(Paths.get(commandLine.getOptionValue(Constants.PARAM_TILE_LIST_FILE)))); } if (commandLine.hasOption(Constants.PARAM_PRODUCT_LIST)) { String[] uuids = commandLine.getOptionValues(Constants.PARAM_PRODUCT_UUID_LIST); String[] productNames = commandLine.getOptionValues(Constants.PARAM_PRODUCT_LIST); if (sensorType == SensorType.S2 && (!commandLine.hasOption(Constants.PARAM_DOWNLOAD_STORE) || ProductStore.SCIHUB.toString() .equals(commandLine.getOptionValue(Constants.PARAM_DOWNLOAD_STORE))) && (uuids == null || uuids.length != productNames.length)) { logger.error("For the list of product names a corresponding list of UUIDs has to be given!"); return -1; } for (int i = 0; i < productNames.length; i++) { ProductDescriptor productDescriptor = sensorType == SensorType.S2 ? new SentinelProductDescriptor(productNames[i]) : new LandsatProductDescriptor(productNames[i]); if (uuids != null) { productDescriptor.setId(uuids[i]); } products.add(productDescriptor); } } else if (commandLine.hasOption(Constants.PARAM_PRODUCT_LIST_FILE)) { for (String line : Files .readAllLines(Paths.get(commandLine.getOptionValue(Constants.PARAM_PRODUCT_LIST_FILE)))) { products.add(sensorType == SensorType.S2 ? new SentinelProductDescriptor(line) : new LandsatProductDescriptor(line)); } } double clouds; if (commandLine.hasOption(Constants.PARAM_CLOUD_PERCENTAGE)) { clouds = Double.parseDouble(commandLine.getOptionValue(Constants.PARAM_CLOUD_PERCENTAGE)); } else { clouds = Constants.DEFAULT_CLOUD_PERCENTAGE; } String sensingStart; if (commandLine.hasOption(Constants.PARAM_START_DATE)) { String dateString = commandLine.getOptionValue(Constants.PARAM_START_DATE); LocalDate startDate = LocalDate.parse(dateString, DateTimeFormatter.ISO_DATE); long days = ChronoUnit.DAYS.between(startDate, LocalDate.now()); sensingStart = String.format(Constants.PATTERN_START_DATE, days); } else { sensingStart = Constants.DEFAULT_START_DATE; } String sensingEnd; if (commandLine.hasOption(Constants.PARAM_END_DATE)) { String dateString = commandLine.getOptionValue(Constants.PARAM_END_DATE); LocalDate endDate = LocalDate.parse(dateString, DateTimeFormatter.ISO_DATE); long days = ChronoUnit.DAYS.between(endDate, LocalDate.now()); sensingEnd = String.format(Constants.PATTERN_START_DATE, days); } else { sensingEnd = Constants.DEFAULT_END_DATE; } int limit; if (commandLine.hasOption(Constants.PARAM_RESULTS_LIMIT)) { limit = Integer.parseInt(commandLine.getOptionValue(Constants.PARAM_RESULTS_LIMIT)); } else { limit = Constants.DEFAULT_RESULTS_LIMIT; } if (commandLine.hasOption(Constants.PARAM_DOWNLOAD_STORE)) { String value = commandLine.getOptionValue(Constants.PARAM_DOWNLOAD_STORE); if (downloader instanceof SentinelProductDownloader) { ((SentinelProductDownloader) downloader) .setDownloadStore(Enum.valueOf(ProductStore.class, value)); logger.info("Products will be downloaded from %s", value); } else { logger.warn("Argument --store will be ignored for Landsat8"); } } downloader.shouldCompress(commandLine.hasOption(Constants.PARAM_FLAG_COMPRESS)); downloader.shouldDeleteAfterCompression(commandLine.hasOption(Constants.PARAM_FLAG_DELETE)); if (commandLine.hasOption(Constants.PARAM_FILL_ANGLES)) { if (downloader instanceof SentinelProductDownloader) { ((SentinelProductDownloader) downloader) .setFillMissingAnglesMethod(Enum.valueOf(FillAnglesMethod.class, commandLine.hasOption(Constants.PARAM_FILL_ANGLES) ? commandLine.getOptionValue(Constants.PARAM_FILL_ANGLES).toUpperCase() : FillAnglesMethod.NONE.name())); } else { logger.warn("Argument --ma will be ignored for Landsat8"); } } int numPoints = areaOfInterest.getNumPoints(); tiles = tiles.stream().map(t -> t.startsWith("T") ? t.substring(1) : t).collect(Collectors.toSet()); if (products.size() == 0 && numPoints == 0 && tileMap.getCount() > 0) { Rectangle2D rectangle2D = tileMap.boundingBox(tiles); areaOfInterest.append(rectangle2D.getX(), rectangle2D.getY()); areaOfInterest.append(rectangle2D.getMaxX(), rectangle2D.getY()); areaOfInterest.append(rectangle2D.getMaxX(), rectangle2D.getMaxY()); areaOfInterest.append(rectangle2D.getX(), rectangle2D.getMaxY()); areaOfInterest.append(rectangle2D.getX(), rectangle2D.getY()); } numPoints = areaOfInterest.getNumPoints(); if (products.size() == 0 && numPoints > 0) { String searchUrl; AbstractSearch searchProvider; logger.debug("No product provided, searching on the AOI"); if (sensorType == SensorType.L8) { logger.debug("Search will be done for Landsat"); searchUrl = props.getProperty(Constants.PROPERTY_NAME_LANDSAT_SEARCH_URL, Constants.PROPERTY_NAME_DEFAULT_LANDSAT_SEARCH_URL); if (!NetUtils.isAvailable(searchUrl)) { logger.warn(searchUrl + " is not available!"); } searchProvider = new LandsatSearch(searchUrl); if (commandLine.hasOption(Constants.PARAM_START_DATE)) { searchProvider.setSensingStart(commandLine.getOptionValue(Constants.PARAM_START_DATE)); } if (commandLine.hasOption(Constants.PARAM_END_DATE)) { searchProvider.setSensingEnd(commandLine.getOptionValue(Constants.PARAM_END_DATE)); } if (commandLine.hasOption(Constants.PARAM_TILE_LIST)) { searchProvider.setTiles(tiles); } ((LandsatSearch) searchProvider).limit(limit); } else if (!commandLine.hasOption(Constants.PARAM_FLAG_SEARCH_AWS)) { logger.debug("Search will be done on SciHub"); searchUrl = props.getProperty(Constants.PROPERTY_NAME_SEARCH_URL, Constants.PROPERTY_DEFAULT_SEARCH_URL); if (!NetUtils.isAvailable(searchUrl)) { logger.warn(searchUrl + " is not available!"); searchUrl = props.getProperty(Constants.PROPERTY_NAME_SEARCH_URL_SECONDARY, Constants.PROPERTY_DEFAULT_SEARCH_URL_SECONDARY); } searchProvider = new SciHubSearch(searchUrl); SciHubSearch search = (SciHubSearch) searchProvider; if (user != null && !user.isEmpty() && pwd != null && !pwd.isEmpty()) { search = search.auth(user, pwd); } String interval = "[" + sensingStart + " TO " + sensingEnd + "]"; search.filter(Constants.SEARCH_PARAM_INTERVAL, interval).limit(limit); if (commandLine.hasOption(Constants.PARAM_RELATIVE_ORBIT)) { search.filter(Constants.SEARCH_PARAM_RELATIVE_ORBIT_NUMBER, commandLine.getOptionValue(Constants.PARAM_RELATIVE_ORBIT)); } } else { logger.debug("Search will be done on AWS"); searchUrl = props.getProperty(Constants.PROPERTY_NAME_AWS_SEARCH_URL, Constants.PROPERTY_DEFAULT_AWS_SEARCH_URL); searchProvider = new AmazonSearch(searchUrl); searchProvider.setTiles(tiles); Calendar calendar = Calendar.getInstance(); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); calendar.add(Calendar.DAY_OF_MONTH, Integer.parseInt(sensingStart.replace("NOW", "").replace("DAY", ""))); searchProvider.setSensingStart(dateFormat.format(calendar.getTime())); calendar = Calendar.getInstance(); String endOffset = sensingEnd.replace("NOW", "").replace("DAY", ""); int offset = endOffset.isEmpty() ? 0 : Integer.parseInt(endOffset); calendar.add(Calendar.DAY_OF_MONTH, offset); searchProvider.setSensingEnd(dateFormat.format(calendar.getTime())); if (commandLine.hasOption(Constants.PARAM_RELATIVE_ORBIT)) { searchProvider.setOrbit( Integer.parseInt(commandLine.getOptionValue(Constants.PARAM_RELATIVE_ORBIT))); } } if (searchProvider.getTiles() == null || searchProvider.getTiles().size() == 0) { searchProvider.setAreaOfInterest(areaOfInterest); } searchProvider.setClouds(clouds); products = searchProvider.execute(); } else { logger.debug("Product name(s) present, no additional search will be performed."); } if (downloader instanceof SentinelProductDownloader) { ((SentinelProductDownloader) downloader).setFilteredTiles(tiles, commandLine.hasOption(Constants.PARAM_FLAG_UNPACKED)); } downloader.setProgressListener(batchProgressListener); downloader.setFileProgressListener(fileProgressListener); retCode = downloader.downloadProducts(products); } return retCode; }