List of usage examples for org.apache.commons.net.ftp FTPFile getTimestamp
public Calendar getTimestamp()
From source file:org.codelibs.fess.crawler.client.ftp.FtpClient.java
protected ResponseData getResponseData(final String uri, final boolean includeContent) { final ResponseData responseData = new ResponseData(); FTPClient client = null;/*w ww .j a v a 2s . c o m*/ try { responseData.setMethod(Constants.GET_METHOD); final FtpInfo ftpInfo = new FtpInfo(uri); responseData.setUrl(ftpInfo.toUrl()); client = getClient(ftpInfo); FTPFile file = null; client.changeWorkingDirectory(ftpInfo.getParent()); validateRequest(client); if (ftpInfo.getName() == null) { // root directory final Set<RequestData> requestDataSet = new HashSet<>(); if (includeContent) { try { final FTPFile[] files = client.listFiles(ftpInfo.getParent(), FTPFileFilters.NON_NULL); validateRequest(client); for (final FTPFile f : files) { final String chileUri = ftpInfo.toUrl(f.getName()); requestDataSet.add(RequestDataBuilder.newRequestData().get().url(chileUri).build()); } } catch (final IOException e) { throw new CrawlingAccessException("Could not access " + uri, e); } } ftpClientQueue.offer(client); throw new ChildUrlsException(requestDataSet, this.getClass().getName() + "#getResponseData(String, boolean)"); } final FTPFile[] files = client.listFiles(null, FTPFileFilters.NON_NULL); validateRequest(client); for (final FTPFile f : files) { if (ftpInfo.getName().equals(f.getName())) { file = f; break; } } if (file == null) { responseData.setHttpStatusCode(Constants.NOT_FOUND_STATUS_CODE); responseData.setCharSet(charset); responseData.setContentLength(0); } else if (file.isFile()) { responseData.setHttpStatusCode(Constants.OK_STATUS_CODE); responseData.setCharSet(Constants.UTF_8); responseData.setLastModified(file.getTimestamp().getTime()); // check file size responseData.setContentLength(file.getSize()); checkMaxContentLength(responseData); if (includeContent) { File tempFile = null; File outputFile = null; try { tempFile = File.createTempFile("ftp-", ".tmp"); try (OutputStream out = new BufferedOutputStream(new FileOutputStream(tempFile))) { if (!client.retrieveFile(ftpInfo.getName(), out)) { throw new CrawlingAccessException("Failed to retrieve: " + ftpInfo.toUrl()); } } final MimeTypeHelper mimeTypeHelper = crawlerContainer.getComponent("mimeTypeHelper"); try (InputStream is = new FileInputStream(tempFile)) { responseData.setMimeType(mimeTypeHelper.getContentType(is, file.getName())); } catch (final Exception e) { responseData.setMimeType(mimeTypeHelper.getContentType(null, file.getName())); } if (contentLengthHelper != null) { final long maxLength = contentLengthHelper.getMaxLength(responseData.getMimeType()); if (responseData.getContentLength() > maxLength) { throw new MaxLengthExceededException( "The content length (" + responseData.getContentLength() + " byte) is over " + maxLength + " byte. The url is " + uri); } } responseData.setCharSet(geCharSet(tempFile)); if (tempFile.length() < maxCachedContentSize) { try (InputStream contentStream = new BufferedInputStream( new FileInputStream(tempFile))) { responseData.setResponseBody(InputStreamUtil.getBytes(contentStream)); } } else { outputFile = File.createTempFile("crawler-FileSystemClient-", ".out"); CopyUtil.copy(tempFile, outputFile); responseData.setResponseBody(outputFile, true); } } catch (final Exception e) { logger.warn("I/O Exception.", e); responseData.setHttpStatusCode(Constants.SERVER_ERROR_STATUS_CODE); } finally { if (tempFile != null && !tempFile.delete()) { logger.warn("Could not delete " + tempFile.getAbsolutePath()); } if (outputFile != null && !outputFile.delete()) { logger.warn("Could not delete " + outputFile.getAbsolutePath()); } } } } else if (file.isDirectory()) { final Set<RequestData> requestDataSet = new HashSet<>(); if (includeContent) { try { final FTPFile[] ftpFiles = client.listFiles(ftpInfo.getName(), FTPFileFilters.NON_NULL); validateRequest(client); for (final FTPFile f : ftpFiles) { final String chileUri = ftpInfo.toUrl(f.getName()); requestDataSet.add(RequestDataBuilder.newRequestData().get().url(chileUri).build()); } } catch (final IOException e) { throw new CrawlingAccessException("Could not access " + uri, e); } } ftpClientQueue.offer(client); throw new ChildUrlsException(requestDataSet, this.getClass().getName() + "#getResponseData(String, boolean)"); } else { responseData.setHttpStatusCode(Constants.NOT_FOUND_STATUS_CODE); responseData.setCharSet(charset); responseData.setContentLength(0); } ftpClientQueue.offer(client); } catch (final CrawlerSystemException e) { IOUtils.closeQuietly(responseData); throw e; } catch (final Exception e) { IOUtils.closeQuietly(responseData); throw new CrawlingAccessException("Could not access " + uri, e); } return responseData; }
From source file:org.covito.kit.file.support.FtpFileServiceImpl.java
/** * {@inheritDoc}//w ww . j a v a 2 s . c om * * @author covito * @param path * @return */ @Override public FileMeta getFileInfo(String path) { init(); if (path == null || path.length() == 0) { return null; } FileMeta file = new FileMeta(); FTPClient client = getConnect(); if (!dealDocPath(client, path, false)) { throw new FileServiceException("path not exist!"); } ByteArrayOutputStream bos = new ByteArrayOutputStream(); try { boolean sucess = client.retrieveFile(getMetaPath(path), bos); if (!sucess) { log.error(client.getReplyString()); throw new FileServiceException("path is not exist!"); } } catch (IOException e) { e.printStackTrace(); throw new FileServiceException("path is not exist!"); } JSONObject json = JSON.parseObject(bos.toString()); Map<String, String> meta = JSON.parseObject(json.toString(), Map.class); file.setMeta(meta); file.setFileName(meta.get(FileMeta.KEY_FILENAME)); try { FTPFile ftpfile = client.mlistFile(rootWorkingDirectory + "/" + path); file.setCreateTime(new Date(ftpfile.getTimestamp().getTimeInMillis())); file.setFileSize(ftpfile.getSize()); } catch (IOException e1) { e1.printStackTrace(); } return file; }
From source file:org.eurocarbdb.action.admin.AbstractDownloadAction.java
/************************************************ */*from w w w . j a va 2s. c o m*/ * Downloads the named remote file to the given output stream. * This method blocks until download is complete, however * information about download progress may be obtained * asynchronously (in another thread) via the following methods: * <ul> * <li>getMillisecsElapsed</li> * <li>getPercentComplete</li> * <li>getDownloadSpeed</li> * </ul> * * @see FTP_Client#statFile */ protected void download(String filename, OutputStream out) throws IOException, FTPConnectionClosedException { if (ftpClient == null) throw new RuntimeException("client not yet connected!!!"); startTime = System.currentTimeMillis(); outStream = out; try { ftpClient.setPassiveMode(true); ftpClient.setBinaryMode(true); // work out file mtime and size in bytes FTPFile filestat = ftpClient.statFile(filename); // mjh:TODO check not null here this.fileSize = filestat.getSize(); this.fileTimestamp = filestat.getTimestamp(); if (log.isDebugEnabled()) { log.debug("File size is " + fileSize + ", last modified " + fileTimestamp); } // open input stream this.inStream = ftpClient.retrieveFileStream(filename); byte[] buffer = new byte[BUFFER_SIZE]; while (true) { // read as many bytes as are available, up to the length of the buffer. int available = inStream.available(); int bytes = (available <= buffer.length) ? available : buffer.length; // the call blocks until data is read int bytes_read = inStream.read(buffer, 0, bytes); if (bytes_read == -1 || bytesDownloaded == fileSize) break; if (bytes_read == 0) continue; //if ( log.isTraceEnabled() ) // log.trace("read " + bytes_read + " bytes"); bytesDownloaded += bytes_read; outStream.write(buffer, 0, bytes_read); } log.debug("finished transfer, ending FTP transaction"); try { inStream.close(); } catch (IOException ioe) { log.warn("Caught exception closing input streams:", ioe); } if (!ftpClient.completePendingCommand()) throw new IOException("completePendingCommand returned false, " + "probably indicating a failed download " + "(unspecified reason)"); log.info("download completed successfully"); } catch (FTPConnectionClosedException ioe) { log.warn("FTP server closed connection: ", ioe); ioe.printStackTrace(); closeConnection(); throw ioe; } catch (IOException ioe) { log.warn("Caught IO exception during FTP: ", ioe); ioe.printStackTrace(); closeConnection(); throw ioe; } }
From source file:org.fao.fenix.ftp.FTPTask.java
private void validateAndImportFile(FTPFile[] ftpFiles, String datasetDirectoryName) { for (FTPFile file : ftpFiles) { // if (!file.isFile() && !file.getName().endsWith(".csv")) { // continue; // }// ww w . ja v a 2 s .co m Date fileDate = file.getTimestamp().getTime(); String formattedFileDate = df.format(fileDate); Date today = new java.util.Date(); String formattedTodayDate = df.format(today); if (formattedFileDate.equals(formattedTodayDate)) { String datasetCode = ""; String tempTableName = ""; if (datasetDirectoryName.equalsIgnoreCase("indices")) { datasetCode = "AMIS_IGC_DAILY_INDICATORS"; tempTableName = "igc_daily_indicators_temp"; } jdbcConnector.openConnection(); // Check the last modified date of the database String lastModifiedDate = amisDataImporter.getLastModifiedDate(datasetCode); System.out.println("=== amis-ftp === FTPTask: validateAndImportFile: == INFO == formattedFileDate " + formattedFileDate); System.out.println("=== amis-ftp === FTPTask: validateAndImportFile: == INFO == lastModifiedDate " + lastModifiedDate); if (!formattedFileDate.equals(lastModifiedDate)) { //Retrieve the file from the FTP and place in download directory retrieveFileFromFTP(formattedTodayDate, file); // connect to DB and get metadata for the dataset in order validate the csv Boolean isValidCSV = csvBasicValidator.validateCSV( getPath() + File.separator + formattedTodayDate + "_" + file.getName(), datasetCode); if (isValidCSV) { System.out.println( "=== amis-ftp === FTPTask: validateAndImportFile: == INFO == CSV is Validated ... proceed to import"); int expectedRows = csvBasicValidator.getLastLineNumber( getPath() + File.separator + formattedTodayDate + "_" + file.getName()); //set date format amisDataImporter.setPostgresDateFormat(); int rowsInsertedIntoTemp = amisDataImporter.importDataFromCSVIntoTempTable( formattedTodayDate + "_" + file.getName(), expectedRows, tempTableName); if (rowsInsertedIntoTemp == expectedRows) { //import the data into the final data table int rowsInserted = amisDataImporter.importDataFromCSVIntoDataTable( formattedTodayDate + "_" + file.getName(), datasetCode, expectedRows); System.out.println( "=== amis-ftp === FTPTask: validateAndImportFile: == INFO == Data Table: rowsInserted into " + datasetCode + " = " + rowsInserted + " (expected = " + expectedRows + ")"); //reset date format amisDataImporter.resetPostgresDateFormat(); int lastModifiedDateUpdatedRows = amisDataImporter.updateLastModifiedDate(datasetCode); System.out.println( "=== amis-ftp === FTPTask: validateAndImportFile: == INFO == CustomDataset Table: lastModifiedDateUpdated Executed affected " + lastModifiedDateUpdatedRows + " rows"); // Empty the download folder boolean isUploadedFileDeleted = emptyDownloadDirectory(); System.out.println( "=== amis-ftp === FTPTask: validateAndImportFile: == INFO == Is the uploaded file deleted from the 'download' directory: " + isUploadedFileDeleted); //Send Email once successfully updated! sendOnSuccessEmail(datasetCode); } else { //reset date format amisDataImporter.resetPostgresDateFormat(); System.out.println( "=== amis-ftp === FTPTask: validateAndImportFile: == ERROR == The number of rows in the CSV (" + expectedRows + ") do not match how many rows where inserted into the igc_daily_indicators_temp (temporary) database table (" + rowsInsertedIntoTemp + ")... so the import into the actual database table DID NOT happen "); String message = "FTPTask: validateAndImportFile(): The number of rows in the CSV (" + expectedRows + ") do not match how many rows where inserted into the igc_daily_indicators_temp (temporary) database table (" + rowsInsertedIntoTemp + ")... so the import into the actual database table DID NOT happen "; emailSender.sendEMailToDevelopers(emailErrorSubject, message); //emailSender.sendEMailToContacts(emailErrorSubject, message, false); } } csvBasicValidator.closeCSVParser(); } else { System.out.println( "=== amis-ftp === FTPTask: validateAndImportFile: == WARNING == The dataset was last modified on " + lastModifiedDate + ", which matches the date of the FTP file (i.e." + formattedFileDate + ") - so there is no need to re-upload"); } jdbcConnector.closeConnection(); } else { System.out.println( "=== amis-ftp === FTPTask: validateAndImportFile: == WARNING == No new files in the FTP for " + formattedTodayDate); } } }
From source file:org.ikasan.connector.ftp.net.FileTransferProtocolClient.java
/** * Constructing a <code>ClientListEntry</code> object from an * <code>FTPFile</code> object. This is a direct map with some formatting * changes./*from w w w . j a v a 2 s .com*/ * * @param ftpFile The <code>FTPFile</code> to map to a * <code>ClientListEntry</code> * @param fileUri The URI of the underlying file for the particular * <code>FTPFile</code> * @return ClientListEntry */ private ClientListEntry convertFTPFileToClientListEntry(FTPFile ftpFile, URI fileUri, String currentDir) { ClientListEntry clientListEntry = new ClientListEntry(); clientListEntry.setUri(fileUri); clientListEntry.setName(ftpFile.getName()); clientListEntry.setFullPath(currentDir + System.getProperty("file.separator") + ftpFile.getName()); clientListEntry.setClientId(null); // Can't distinguish between Last Accessed and Last Modified clientListEntry.setDtLastAccessed(ftpFile.getTimestamp().getTime()); clientListEntry.setDtLastModified(ftpFile.getTimestamp().getTime()); clientListEntry.setSize(ftpFile.getSize()); clientListEntry.isDirectory(ftpFile.isDirectory()); clientListEntry.isLink(ftpFile.isSymbolicLink()); clientListEntry.setLongFilename(ftpFile.getRawListing()); clientListEntry.setAtime(ftpFile.getTimestamp().getTime().getTime()); clientListEntry.setMtime(ftpFile.getTimestamp().getTime().getTime()); clientListEntry.setAtimeString(ftpFile.getTimestamp().toString()); clientListEntry.setMtimeString(ftpFile.getTimestamp().toString()); // clientListEntry.setFlags(); clientListEntry.setGid(ftpFile.getGroup()); clientListEntry.setUid(ftpFile.getUser()); // TODO might be able to ask which permissions it has and build an int // and String from there // clientListEntry.setPermissions(); // clientListEntry.setPermissionsString(); // No extended information clientListEntry.setExtended(null); return clientListEntry; }
From source file:org.jevis.commons.driver.DataSourceHelper.java
public static List<String> getFTPMatchedFileNames(FTPClient fc, DateTime lastReadout, String filePath) { filePath = filePath.replace("\\", "/"); String[] pathStream = getPathTokens(filePath); String startPath = ""; if (filePath.startsWith("/")) { startPath = "/"; }/* www .j ava 2s. c om*/ List<String> folderPathes = getMatchingPathes(startPath, pathStream, new ArrayList<String>(), fc, lastReadout, new DateTimeFormatterBuilder()); // System.out.println("foldersize,"+folderPathes.size()); List<String> fileNames = new ArrayList<String>(); if (folderPathes.isEmpty()) { org.apache.log4j.Logger.getLogger(DataSourceHelper.class).log(org.apache.log4j.Level.ERROR, "Cant find suitable folder on the device"); return fileNames; } // String fileName = null; String fileNameScheme = pathStream[pathStream.length - 1]; String currentfolder = null; try { for (String folder : folderPathes) { // fc.changeWorkingDirectory(folder); // System.out.println("currentFolder,"+folder); currentfolder = folder; // for (FTPFile file : fc.listFiles(folder)) { // System.out.println(file.getName()); // } fc.changeWorkingDirectory(folder); for (FTPFile file : fc.listFiles()) { // org.apache.log4j.Logger.getLogger(Launcher.class.getName()).log(org.apache.log4j.Level.ALL, "CurrentFileName: " + fileName); // fileName = removeFoler(fileName, folder); if (file.getTimestamp().compareTo(lastReadout.toGregorianCalendar()) < 0) { continue; } boolean match = false; System.out.println(file.getName()); if (DataSourceHelper.containsTokens(fileNameScheme)) { boolean matchDate = matchDateString(file.getName(), fileNameScheme); DateTime folderTime = getFileTime(folder + file.getName(), pathStream); boolean isLater = folderTime.isAfter(lastReadout); if (matchDate && isLater) { match = true; } } else { Pattern p = Pattern.compile(fileNameScheme); Matcher m = p.matcher(file.getName()); match = m.matches(); } if (match) { fileNames.add(folder + file.getName()); } } } } catch (IOException ex) { org.apache.log4j.Logger.getLogger(DataSourceHelper.class).log(org.apache.log4j.Level.ERROR, ex.getMessage()); } catch (Exception ex) { org.apache.log4j.Logger.getLogger(DataSourceHelper.class).log(org.apache.log4j.Level.ERROR, "Error while searching a matching file"); org.apache.log4j.Logger.getLogger(DataSourceHelper.class).log(org.apache.log4j.Level.ERROR, "Folder: " + currentfolder); org.apache.log4j.Logger.getLogger(DataSourceHelper.class).log(org.apache.log4j.Level.ERROR, "FileName: " + fileNameScheme); org.apache.log4j.Logger.getLogger(DataSourceHelper.class).log(org.apache.log4j.Level.ERROR, ex.getMessage()); } if (folderPathes.isEmpty()) { org.apache.log4j.Logger.getLogger(DataSourceHelper.class).log(org.apache.log4j.Level.ERROR, "Cant find suitable files on the device"); } // System.out.println("filenamesize"+fileNames.size()); return fileNames; }
From source file:org.madeirahs.shared.provider.FTPProvider.java
/** * Obtains the last modification time of the file. * @param fileName//from ww w . ja v a 2s . com * @return a Calendar object representing the last modification time of the file in the local time zone. */ public Calendar getLastModified(String fileName) { synchronized (ftp) { try { guard.acquire(); } catch (InterruptedException e) { e.printStackTrace(); } try { FTPFile file = ftp.mlistFile(fileName); Calendar time = file.getTimestamp(); time.setTimeZone(TimeZone.getDefault()); return time; } catch (IOException e) { e.printStackTrace(); } } return null; }
From source file:org.martin.ftp.util.RemoteComparator.java
@Override public int compare(FTPFile o1, FTPFile o2) { if (null != option) switch (option) { case DATE: if (order == SortOption.UPWARD) return (int) (o1.getTimestamp().getTimeInMillis() - o2.getTimestamp().getTimeInMillis()); else// w w w . j a v a2 s .co m return (int) (o2.getTimestamp().getTimeInMillis() - o1.getTimestamp().getTimeInMillis()); case FORMAT: String file1Format = Utilities.getFormat(o1.getName()); String file2Format = Utilities.getFormat(o2.getName()); if (order == SortOption.UPWARD) if (o1.isDirectory() || o2.isDirectory()) { if (o1.isDirectory() && o2.isDirectory()) return idDirectory - idDirectory; else if (o1.isDirectory() && !o2.isDirectory()) return idDirectory - idFile; else return idFile - idDirectory; } else return file1Format.compareToIgnoreCase(file2Format); else if (o1.isDirectory() || o2.isDirectory()) { if (o1.isDirectory() && o2.isDirectory()) return idDirectory - idDirectory; else if (o1.isDirectory() && !o2.isDirectory()) return idFile - idDirectory; else return idDirectory - idFile; } else return file2Format.compareToIgnoreCase(file1Format); case NAME: if (order == SortOption.UPWARD) return o1.getName().compareToIgnoreCase(o2.getName()); else return o2.getName().compareToIgnoreCase(o1.getName()); case SIZE: if (order == SortOption.UPWARD) return (int) (o1.getSize() - o2.getSize()); else return (int) (o2.getSize() - o1.getSize()); case TYPE: if ((o1.isDirectory() && o2.isDirectory()) || (!o1.isDirectory() && !o2.isDirectory())) return idDirectory - idDirectory; if (order == SortOption.UPWARD) if (o1.isDirectory() && !o2.isDirectory()) return idDirectory - idFile; else return idFile - idDirectory; else if (o1.isDirectory() && !o2.isDirectory()) return idFile - idDirectory; else return idDirectory - idFile; default: return 0; } return 0; }
From source file:org.martin.ftp.util.SearchComparator.java
@Override public int compare(FileFiltering o1, FileFiltering o2) { FTPFile fileO1 = o1.getFile(); FTPFile fileO2 = o2.getFile();/*from w w w . jav a2 s . c o m*/ if (null != option) switch (option) { case DATE: if (order == SortOption.UPWARD) return (int) (fileO1.getTimestamp().getTimeInMillis() - fileO2.getTimestamp().getTimeInMillis()); else return (int) (fileO2.getTimestamp().getTimeInMillis() - fileO1.getTimestamp().getTimeInMillis()); case FORMAT: String file1Format = Utilities.getFormat(fileO1.getName()); String file2Format = Utilities.getFormat(fileO2.getName()); if (order == SortOption.UPWARD) if (fileO1.isDirectory() || fileO2.isDirectory()) { if (fileO1.isDirectory() && fileO2.isDirectory()) return idDirectory - idDirectory; else if (fileO1.isDirectory() && !fileO2.isDirectory()) return idDirectory - idFile; else return idFile - idDirectory; } else return file1Format.compareToIgnoreCase(file2Format); else if (fileO1.isDirectory() || fileO2.isDirectory()) { if (fileO1.isDirectory() && fileO2.isDirectory()) return idDirectory - idDirectory; else if (fileO1.isDirectory() && !fileO2.isDirectory()) return idFile - idDirectory; else return idDirectory - idFile; } else return file2Format.compareToIgnoreCase(file1Format); case NAME: if (order == SortOption.UPWARD) return fileO1.getName().compareToIgnoreCase(fileO2.getName()); else return fileO2.getName().compareToIgnoreCase(fileO1.getName()); case SIZE: if (order == SortOption.UPWARD) return (int) (fileO1.getSize() - fileO2.getSize()); else return (int) (fileO2.getSize() - fileO1.getSize()); case TYPE: if ((fileO1.isDirectory() && fileO2.isDirectory()) || (!fileO1.isDirectory() && !fileO2.isDirectory())) return idDirectory - idDirectory; if (order == SortOption.UPWARD) if (fileO1.isDirectory() && !fileO2.isDirectory()) return idDirectory - idFile; else return idFile - idDirectory; else if (fileO1.isDirectory() && !fileO2.isDirectory()) return idFile - idDirectory; else return idDirectory - idFile; default: return 0; } return 0; }
From source file:org.mule.transport.ftp.FtpMuleMessageFactory.java
@Override protected void addProperties(DefaultMuleMessage message, Object transportMessage) throws Exception { super.addProperties(message, transportMessage); FTPFile file = (FTPFile) transportMessage; message.setInboundProperty(FileConnector.PROPERTY_ORIGINAL_FILENAME, file.getName()); message.setInboundProperty(FileConnector.PROPERTY_FILE_SIZE, file.getSize()); message.setInboundProperty(FileConnector.PROPERTY_FILE_TIMESTAMP, file.getTimestamp()); }