List of usage examples for org.apache.hadoop.fs FileSystem isDirectory
@Deprecated public boolean isDirectory(Path f) throws IOException
From source file:hydrograph.engine.spark.datasource.utils.SFTPUtil.java
License:Apache License
public void upload(RunFileTransferEntity runFileTransferEntity) { log.debug("Start SFTPUtil upload"); JSch jsch = new JSch(); Session session = null;// w w w . j ava 2s . c om Channel channel = null; ChannelSftp sftpChannel = null; ZipInputStream zip = null; FileInputStream fin = null; int retryAttempt = 0; int i; File filecheck = new File(runFileTransferEntity.getInputFilePath()); if (runFileTransferEntity.getFailOnError()) if (!(filecheck.isFile() || filecheck.isDirectory()) && !(runFileTransferEntity.getInputFilePath().contains("hdfs://"))) { log.error("invalid input file path,Please provide valid file path"); throw new SFTPUtilException("Invalid input file path"); } if (runFileTransferEntity.getRetryAttempt() == 0) retryAttempt = 1; else retryAttempt = runFileTransferEntity.getRetryAttempt(); for (i = 0; i < retryAttempt; i++) { try { log.info("connection attempt: " + (i + 1)); if (runFileTransferEntity.getPrivateKeyPath() != null) { jsch.addIdentity(runFileTransferEntity.getPrivateKeyPath()); } log.debug("connection details: " + "/n" + "Username: " + runFileTransferEntity.getUserName() + "/n" + "HostName " + runFileTransferEntity.getHostName() + "/n" + "Portno" + runFileTransferEntity.getPortNo()); session = jsch.getSession(runFileTransferEntity.getUserName(), runFileTransferEntity.getHostName(), runFileTransferEntity.getPortNo()); session.setConfig("PreferredAuthentications", "publickey,keyboard-interactive,password"); session.setConfig("StrictHostKeyChecking", "no"); if (runFileTransferEntity.getPassword() != null) { session.setPassword(runFileTransferEntity.getPassword()); } if (runFileTransferEntity.getTimeOut() > 0) { session.setTimeout(runFileTransferEntity.getTimeOut()); } session.connect(); channel = session.openChannel("sftp"); channel.connect(); sftpChannel = (ChannelSftp) channel; sftpChannel.setFilenameEncoding(runFileTransferEntity.getEncoding()); sftpChannel .cd(runFileTransferEntity.getOutFilePath().replaceAll(Matcher.quoteReplacement("\\"), "/")); if (runFileTransferEntity.getInputFilePath().contains("hdfs://")) { log.debug("in hdfs file system transfer"); String inputPath = runFileTransferEntity.getInputFilePath(); File f = new File("/tmp"); if (!f.exists()) f.mkdir(); String s1 = inputPath.substring(7, inputPath.length()); String s2 = s1.substring(0, s1.indexOf("/")); Configuration conf = new Configuration(); conf.set("fs.defaultFS", "hdfs://" + s2); FileSystem hdfsFileSystem = FileSystem.get(conf); Path local = new Path("/tmp"); String s = inputPath.substring(7, inputPath.length()); String hdfspath = s.substring(s.indexOf("/"), s.length()); File dir = new File(hdfspath); if (hdfsFileSystem.isDirectory(new Path(hdfspath))) { log.debug("in hdfs file system folder path"); InputStream is = null; OutputStream os = null; String localDirectory = hdfspath.substring(hdfspath.lastIndexOf("/") + 1); FileStatus[] fileStatus = hdfsFileSystem .listStatus(new Path(runFileTransferEntity.getInputFilePath())); Path[] paths = FileUtil.stat2Paths(fileStatus); File dirs = null; try { String folderName = hdfspath.substring(hdfspath.lastIndexOf("/") + 1); DateFormat df = new SimpleDateFormat("dd-MM-yyyy"); String dateWithoutTime = df.format(new Date()).toString(); java.util.Random ran = new Random(); String tempFolder = "ftp_sftp_" + System.nanoTime() + "_" + ran.nextInt(1000); dirs = new File("/tmp/" + tempFolder); boolean success = dirs.mkdirs(); for (Path file : paths) { is = hdfsFileSystem.open(file); os = new BufferedOutputStream( new FileOutputStream(dirs + "" + File.separatorChar + file.getName())); IOUtils.copyBytes(is, os, conf); } try { sftpChannel.cd(folderName); } catch (SftpException e) { sftpChannel.mkdir(folderName); sftpChannel.cd(folderName); } for (File files : dirs.listFiles()) { if (files.isFile()) if (files.isFile()) { sftpChannel.put(new BufferedInputStream(new FileInputStream(files)), files.getName()); } } } catch (IOException e) { log.error("error while transferring file", e); throw new SFTPUtilException(e); } finally { IOUtils.closeStream(is); IOUtils.closeStream(os); if (dirs != null) { FileUtils.deleteDirectory(dirs); } } } else { log.debug("File transfer in normal mode"); Path hdfs = new Path(hdfspath); hdfsFileSystem.copyToLocalFile(false, hdfs, local); int index = inputPath.replaceAll(Matcher.quoteReplacement("\\"), "/").lastIndexOf('/'); String file_name = runFileTransferEntity.getInputFilePath().substring(index + 1); fin = new FileInputStream("/tmp/" + file_name); sftpChannel.cd(runFileTransferEntity.getOutFilePath() .replaceAll(Matcher.quoteReplacement("\\"), "/")); sftpChannel.put(new BufferedInputStream(fin), file_name); i = i + 1; fin.close(); } } else { java.nio.file.Path file = new File(runFileTransferEntity.getInputFilePath()).toPath(); if (Files.isDirectory(file)) { log.debug("Folder transfer in SFTP"); File f = new File(file.toAbsolutePath().toString()); String folderName = new File(runFileTransferEntity.getInputFilePath()).getName(); sftpChannel.cd(runFileTransferEntity.getOutFilePath() .replaceAll(Matcher.quoteReplacement("\\"), "/")); try { sftpChannel.cd(folderName); } catch (SftpException e) { log.error( "changing the directory but the folde location not found,so create a new directory"); sftpChannel.cd(runFileTransferEntity.getOutFilePath() .replaceAll(Matcher.quoteReplacement("\\"), "/")); sftpChannel.mkdir(folderName); sftpChannel.cd(folderName); } for (File files : f.listFiles()) { if (files.isFile()) sftpChannel.put(new BufferedInputStream(new FileInputStream(files)), files.getName()); } } else { int index = runFileTransferEntity.getInputFilePath() .replaceAll(Matcher.quoteReplacement("\\"), "/").lastIndexOf('/'); String file_name = runFileTransferEntity.getInputFilePath().substring(index + 1); fin = new FileInputStream(runFileTransferEntity.getInputFilePath()); sftpChannel.cd(runFileTransferEntity.getOutFilePath() .replaceAll(Matcher.quoteReplacement("\\"), "/")); sftpChannel.put(new BufferedInputStream(fin), file_name); fin.close(); } } } catch (JSchException e) { if (e.getMessage().compareTo("Auth fail") == 0) { log.error("authentication error,please provide valid details", e); if (runFileTransferEntity.getFailOnError()) throw new SFTPUtilException(e.getMessage()); } else { log.error("error while transfering the file and retrying ", e); try { Thread.sleep(runFileTransferEntity.getRetryAfterDuration()); } catch (Exception e1) { log.error("sleep duration for re attemp exception", e1); } continue; } } catch (Exception e) { log.error("Error while transfering the file", e); try { Thread.sleep(runFileTransferEntity.getRetryAfterDuration()); } catch (Exception e1) { log.error("exception while sleep thread", e); } continue; } finally { try { if (fin != null) fin.close(); } catch (IOException ioe) { log.error("error while closing input stream "); } } done = true; break; } if (sftpChannel != null) { sftpChannel.disconnect(); } if (channel != null) { channel.disconnect(); } if (session != null) { session.disconnect(); } if (runFileTransferEntity.getFailOnError() && !done) { log.error("File transfer failed"); throw new SFTPUtilException("File transfer failed"); } else if (!done) { log.error("File transfer failed but mentioned fail on error as false"); } log.debug("Fininished SFTPUtil upload"); }
From source file:io.dataapps.chlorine.hadoop.NewFilesFilter.java
License:Apache License
@Override public boolean accept(Path path) { try {/*from w w w .j a v a2s.c om*/ FileSystem fs = FileSystem.get(conf); if (fs.isDirectory(path)) { return true; } FileStatus file = fs.getFileStatus(path); long time = file.getModificationTime(); boolean result = (time > cutOffTime); return result; } catch (IOException e) { LOG.error(e); return true; } }
From source file:io.divolte.server.hdfs.SessionBinningFileStrategy.java
License:Apache License
public SessionBinningFileStrategy(final ValidatedConfiguration vc, final FileSystem hdfs, final short hdfsReplication, final Schema schema) { sessionTimeoutMillis = vc.configuration().tracking.sessionTimeout.toMillis(); hostString = findLocalHostName();/*from w w w.j a va 2s. c o m*/ instanceNumber = INSTANCE_COUNTER.incrementAndGet(); hdfsWorkingDir = vc.configuration().hdfsFlusher.fileStrategy.asSessionBinningFileStrategy().workingDir; hdfsPublishDir = vc.configuration().hdfsFlusher.fileStrategy.asSessionBinningFileStrategy().publishDir; syncEveryMillis = vc.configuration().hdfsFlusher.fileStrategy .asSessionBinningFileStrategy().syncFileAfterDuration.toMillis(); syncEveryRecords = vc.configuration().hdfsFlusher.fileStrategy .asSessionBinningFileStrategy().syncFileAfterRecords; this.hdfs = hdfs; this.hdfsReplication = hdfsReplication; this.schema = schema; openFiles = Maps.newHashMapWithExpectedSize(10); throwsIoException(() -> { if (!hdfs.isDirectory(new Path(hdfsWorkingDir))) { throw new IOException( "Working directory for in-flight AVRO records does not exist: " + hdfsWorkingDir); } if (!hdfs.isDirectory(new Path(hdfsPublishDir))) { throw new IOException( "Working directory for publishing AVRO records does not exist: " + hdfsPublishDir); } }).ifPresent((e) -> { throw new RuntimeException("Configuration error", e); }); }
From source file:io.druid.storage.hdfs.HdfsDataSegmentFinder.java
License:Apache License
@Override public Set<DataSegment> findSegments(String workingDirPathStr, boolean updateDescriptor) throws SegmentLoadingException { final Set<DataSegment> segments = Sets.newHashSet(); final Path workingDirPath = new Path(workingDirPathStr); FileSystem fs; try {/*from w w w .j a v a 2s.c o m*/ fs = workingDirPath.getFileSystem(config); log.info(fs.getScheme()); log.info("FileSystem URI:" + fs.getUri().toString()); if (!fs.exists(workingDirPath)) { throw new SegmentLoadingException("Working directory [%s] doesn't exist.", workingDirPath); } if (!fs.isDirectory(workingDirPath)) { throw new SegmentLoadingException("Working directory [%s] is not a directory!?", workingDirPath); } final RemoteIterator<LocatedFileStatus> it = fs.listFiles(workingDirPath, true); while (it.hasNext()) { final LocatedFileStatus locatedFileStatus = it.next(); final Path path = locatedFileStatus.getPath(); if (path.getName().endsWith("descriptor.json")) { final Path indexZip; final String descriptorParts[] = path.getName().split("_"); if (descriptorParts.length == 2 && descriptorParts[1].equals("descriptor.json") && org.apache.commons.lang.StringUtils.isNumeric(descriptorParts[0])) { indexZip = new Path(path.getParent(), StringUtils.format("%s_index.zip", descriptorParts[0])); } else { indexZip = new Path(path.getParent(), "index.zip"); } if (fs.exists(indexZip)) { final DataSegment dataSegment = mapper.readValue(fs.open(path), DataSegment.class); log.info("Found segment [%s] located at [%s]", dataSegment.getIdentifier(), indexZip); final Map<String, Object> loadSpec = dataSegment.getLoadSpec(); final String pathWithoutScheme = indexZip.toUri().getPath(); if (!loadSpec.get("type").equals(HdfsStorageDruidModule.SCHEME) || !loadSpec.get("path").equals(pathWithoutScheme)) { loadSpec.put("type", HdfsStorageDruidModule.SCHEME); loadSpec.put("path", pathWithoutScheme); if (updateDescriptor) { log.info("Updating loadSpec in descriptor.json at [%s] with new path [%s]", path, pathWithoutScheme); mapper.writeValue(fs.create(path, true), dataSegment); } } segments.add(dataSegment); } else { throw new SegmentLoadingException( "index.zip didn't exist at [%s] while descripter.json exists!?", indexZip); } } } } catch (IOException e) { throw new SegmentLoadingException(e, "Problems interacting with filesystem[%s].", workingDirPath); } return segments; }
From source file:io.druid.storage.hdfs.HdfsDataSegmentPuller.java
License:Apache License
public FileUtils.FileCopyResult getSegmentFiles(final Path path, final File outDir) throws SegmentLoadingException { final LocalFileSystem localFileSystem = new LocalFileSystem(); try {//from ww w.j ava 2s.c o m final FileSystem fs = path.getFileSystem(config); if (fs.isDirectory(path)) { // -------- directory --------- try { return RetryUtils.retry(new Callable<FileUtils.FileCopyResult>() { @Override public FileUtils.FileCopyResult call() throws Exception { if (!fs.exists(path)) { throw new SegmentLoadingException("No files found at [%s]", path.toString()); } final RemoteIterator<LocatedFileStatus> children = fs.listFiles(path, false); final ArrayList<FileUtils.FileCopyResult> localChildren = new ArrayList<>(); final FileUtils.FileCopyResult result = new FileUtils.FileCopyResult(); while (children.hasNext()) { final LocatedFileStatus child = children.next(); final Path childPath = child.getPath(); final String fname = childPath.getName(); if (fs.isDirectory(childPath)) { log.warn("[%s] is a child directory, skipping", childPath.toString()); } else { final File outFile = new File(outDir, fname); // Actual copy fs.copyToLocalFile(childPath, new Path(outFile.toURI())); result.addFile(outFile); } } log.info("Copied %d bytes from [%s] to [%s]", result.size(), path.toString(), outDir.getAbsolutePath()); return result; } }, shouldRetryPredicate(), DEFAULT_RETRY_COUNT); } catch (Exception e) { throw Throwables.propagate(e); } } else if (CompressionUtils.isZip(path.getName())) { // -------- zip --------- final FileUtils.FileCopyResult result = CompressionUtils.unzip(new ByteSource() { @Override public InputStream openStream() throws IOException { return getInputStream(path); } }, outDir, shouldRetryPredicate(), false); log.info("Unzipped %d bytes from [%s] to [%s]", result.size(), path.toString(), outDir.getAbsolutePath()); return result; } else if (CompressionUtils.isGz(path.getName())) { // -------- gzip --------- final String fname = path.getName(); final File outFile = new File(outDir, CompressionUtils.getGzBaseName(fname)); final FileUtils.FileCopyResult result = CompressionUtils.gunzip(new ByteSource() { @Override public InputStream openStream() throws IOException { return getInputStream(path); } }, outFile); log.info("Gunzipped %d bytes from [%s] to [%s]", result.size(), path.toString(), outFile.getAbsolutePath()); return result; } else { throw new SegmentLoadingException("Do not know how to handle file type at [%s]", path.toString()); } } catch (IOException e) { throw new SegmentLoadingException(e, "Error loading [%s]", path.toString()); } }
From source file:io.druid.storage.hdfs.HdfsFileTimestampVersionFinder.java
License:Apache License
/** * Returns the latest modified file at the uri of interest. * * @param uri Either a directory or a file on HDFS. If it is a file, the parent directory will be searched. * @param pattern A pattern matcher for file names in the directory of interest. Passing `null` results in matching any file in the directory. * * @return The URI of the file with the most recent modified timestamp. *//*from ww w. j a va 2s . c o m*/ @Override public URI getLatestVersion(final URI uri, final Pattern pattern) { final Path path = new Path(uri); try { return RetryUtils.retry(new Callable<URI>() { @Override public URI call() throws Exception { final FileSystem fs = path.getFileSystem(config); if (!fs.exists(path)) { return null; } return mostRecentInDir(fs.isDirectory(path) ? path : path.getParent(), pattern); } }, shouldRetryPredicate(), DEFAULT_RETRY_COUNT); } catch (Exception e) { throw Throwables.propagate(e); } }
From source file:io.prestosql.plugin.hive.metastore.file.FileHiveMetastore.java
License:Apache License
@Override public synchronized void createTable(Table table, PrincipalPrivileges principalPrivileges) { verifyTableNotExists(table.getDatabaseName(), table.getTableName()); Path tableMetadataDirectory = getTableMetadataDirectory(table); // validate table location if (table.getTableType().equals(VIRTUAL_VIEW.name())) { checkArgument(table.getStorage().getLocation().isEmpty(), "Storage location for view must be empty"); } else if (table.getTableType().equals(MANAGED_TABLE.name())) { if (!tableMetadataDirectory.equals(new Path(table.getStorage().getLocation()))) { throw new PrestoException(HIVE_METASTORE_ERROR, "Table directory must be " + tableMetadataDirectory); }//w w w .j a v a 2 s .c om } else if (table.getTableType().equals(EXTERNAL_TABLE.name())) { try { Path externalLocation = new Path(table.getStorage().getLocation()); FileSystem externalFileSystem = hdfsEnvironment.getFileSystem(hdfsContext, externalLocation); if (!externalFileSystem.isDirectory(externalLocation)) { throw new PrestoException(HIVE_METASTORE_ERROR, "External table location does not exist"); } if (isChildDirectory(catalogDirectory, externalLocation)) { throw new PrestoException(HIVE_METASTORE_ERROR, "External table location can not be inside the system metadata directory"); } } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not validate external location", e); } } else { throw new PrestoException(NOT_SUPPORTED, "Table type not supported: " + table.getTableType()); } writeSchemaFile("table", tableMetadataDirectory, tableCodec, new TableMetadata(table), false); for (Entry<String, Collection<HivePrivilegeInfo>> entry : principalPrivileges.getUserPrivileges().asMap() .entrySet()) { setTablePrivileges(new PrestoPrincipal(USER, entry.getKey()), table.getDatabaseName(), table.getTableName(), entry.getValue()); } for (Entry<String, Collection<HivePrivilegeInfo>> entry : principalPrivileges.getRolePrivileges().asMap() .entrySet()) { setTablePrivileges(new PrestoPrincipal(ROLE, entry.getKey()), table.getDatabaseName(), table.getTableName(), entry.getValue()); } }
From source file:io.spring.batch.workflow.configuration.MainFlowConfiguration.java
License:Apache License
@Bean public Partitioner partitioner(FileSystem fileSystem) { return new Partitioner() { @Override//w ww . j a v a 2s .co m public Map<String, ExecutionContext> partition(int gridSize) { Map<String, ExecutionContext> contexts = new HashMap<>(); try { FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/probes"), new PathFilter() { @Override public boolean accept(Path path) { try { return fileSystem.isDirectory(path); } catch (IOException e) { return false; } } }); int count = 0; for (FileStatus fileStatus : fileStatuses) { ExecutionContext executionContext = new ExecutionContext(); executionContext.put("curInputDir", fileStatus.getPath().toString()); contexts.put("dir" + count, executionContext); count++; } } catch (IOException e) { e.printStackTrace(); } return contexts; } }; }
From source file:org.apache.accumulo.server.fs.VolumeUtil.java
License:Apache License
static boolean same(FileSystem fs1, Path dir, FileSystem fs2, Path newDir) throws FileNotFoundException, IOException { // its possible that a user changes config in such a way that two uris point to the same thing. Like hdfs://foo/a/b and hdfs://1.2.3.4/a/b both reference // the same thing because DNS resolves foo to 1.2.3.4. This method does not analyze uris to determine if equivalent, instead it inspects the contents of // what the uris point to. // this code is called infrequently and does not need to be optimized. if (fs1.exists(dir) && fs2.exists(newDir)) { if (!fs1.isDirectory(dir)) throw new IllegalArgumentException("expected " + dir + " to be a directory"); if (!fs2.isDirectory(newDir)) throw new IllegalArgumentException("expected " + newDir + " to be a directory"); HashSet<String> names1 = getFileNames(fs1.listStatus(dir)); HashSet<String> names2 = getFileNames(fs2.listStatus(newDir)); if (names1.equals(names2)) { for (String name : names1) if (!hash(fs1, dir, name).equals(hash(fs2, newDir, name))) return false; return true; }//from www . ja v a 2 s . c o m } return false; }
From source file:org.apache.accumulo.tserver.DirectoryDecommissioner.java
License:Apache License
static boolean same(FileSystem fs1, Path dir, FileSystem fs2, Path newDir) throws FileNotFoundException, IOException { // its possible that a user changes config in such a way that two uris point to the same thing. Like hdfs://foo/a/b and hdfs://1.2.3.4/a/b both reference // the same thing because DNS resolves foo to 1.2.3.4. This method does not analyze uris to determine if equivalent, instead it inspects the contents of // what the uris point to. //this code is called infrequently and does not need to be optimized. if (fs1.exists(dir) && fs2.exists(newDir)) { if (!fs1.isDirectory(dir)) throw new IllegalArgumentException("expected " + dir + " to be a directory"); if (!fs2.isDirectory(newDir)) throw new IllegalArgumentException("expected " + newDir + " to be a directory"); HashSet<String> names1 = getFileNames(fs1.listStatus(dir)); HashSet<String> names2 = getFileNames(fs2.listStatus(newDir)); if (names1.equals(names2)) { for (String name : names1) if (!hash(fs1, dir, name).equals(hash(fs2, newDir, name))) return false; return true; }//from www . j ava2 s .c o m } return false; }