List of usage examples for org.apache.hadoop.fs FileSystem isDirectory
@Deprecated public boolean isDirectory(Path f) throws IOException
From source file:org.apache.ambari.fast_hdfs_resource.Resource.java
License:Apache License
public static void checkResourceParameters(Resource resource, FileSystem dfs) throws IllegalArgumentException, IOException { ArrayList<String> actionsAvailable = new ArrayList<String>(); actionsAvailable.add("create"); actionsAvailable.add("delete"); ArrayList<String> typesAvailable = new ArrayList<String>(); typesAvailable.add("file"); typesAvailable.add("directory"); if (resource.getTarget() == null) throw new IllegalArgumentException("Path to resource in HadoopFs must be filled."); if (resource.getAction() == null || !actionsAvailable.contains(resource.getAction())) throw new IllegalArgumentException("Action is not supported."); if (resource.getType() == null || !typesAvailable.contains(resource.getType())) throw new IllegalArgumentException("Type is not supported."); // Check consistency for ("type":"file" == file in hadoop) if (dfs.isFile(new Path(resource.getTarget())) && !"file".equals(resource.getType())) throw new IllegalArgumentException("Cannot create a directory " + resource.getTarget() + " because file is present on the given path."); // Check consistency for ("type":"directory" == directory in hadoop) else if (dfs.isDirectory(new Path(resource.getTarget())) && !"directory".equals(resource.getType())) throw new IllegalArgumentException("Cannot create a file " + resource.getTarget() + " because directory is present on the given path."); if (resource.getSource() != null) { File source = new File(resource.getSource()); if (source.isFile() && !"file".equals(resource.getType())) throw new IllegalArgumentException("Cannot create a directory " + resource.getTarget() + " because source " + resource.getSource() + "is a file"); else if (source.isDirectory() && !"directory".equals(resource.getType())) throw new IllegalArgumentException("Cannot create a file " + resource.getTarget() + " because source " + resource.getSource() + "is a directory"); }/*w w w . j a va2s . co m*/ }
From source file:org.apache.carbondata.datamap.examples.MinMaxIndexDataMap.java
License:Apache License
@Override public void init(DataMapModel model) throws MemoryException, IOException { Path indexPath = FileFactory.getPath(model.getFilePath()); FileSystem fs = FileFactory.getFileSystem(indexPath); if (!fs.exists(indexPath)) { throw new IOException(String.format("Path %s for MinMax index dataMap does not exist", indexPath)); }/*w w w . j a va2 s. c o m*/ if (!fs.isDirectory(indexPath)) { throw new IOException(String.format("Path %s for MinMax index dataMap must be a directory", indexPath)); } FileStatus[] indexFileStatus = fs.listStatus(indexPath, new PathFilter() { @Override public boolean accept(Path path) { return path.getName().endsWith(".minmaxindex"); } }); this.indexFilePath = new String[indexFileStatus.length]; this.readMinMaxDataMap = new MinMaxIndexBlockDetails[indexFileStatus.length][]; for (int i = 0; i < indexFileStatus.length; i++) { this.indexFilePath[i] = indexFileStatus[i].getPath().toString(); this.readMinMaxDataMap[i] = readJson(this.indexFilePath[i]); } }
From source file:org.apache.carbondata.datamap.lucene.LuceneCoarseGrainDataMap.java
License:Apache License
/** * It is called to load the data map to memory or to initialize it. *///from w w w . j a va 2 s . c o m @Override public void init(DataMapModel dataMapModel) throws MemoryException, IOException { // get this path from file path Path indexPath = FileFactory.getPath(dataMapModel.getFilePath()); LOGGER.info("Lucene index read path " + indexPath.toString()); // get file system , use hdfs file system , realized in solr project FileSystem fs = FileFactory.getFileSystem(indexPath); // check this path valid if (!fs.exists(indexPath)) { String errorMessage = String.format("index directory %s not exists.", indexPath); LOGGER.error(errorMessage); throw new IOException(errorMessage); } if (!fs.isDirectory(indexPath)) { String errorMessage = String.format("error index path %s, must be directory", indexPath); LOGGER.error(errorMessage); throw new IOException(errorMessage); } // open this index path , use HDFS default configuration Directory indexDir = new HdfsDirectory(indexPath, FileFactory.getConfiguration()); IndexReader indexReader = DirectoryReader.open(indexDir); if (indexReader == null) { throw new RuntimeException("failed to create index reader object"); } // create a index searcher object indexSearcher = new IndexSearcher(indexReader); }
From source file:org.apache.druid.storage.hdfs.HdfsDataSegmentFinder.java
License:Apache License
@Override public Set<DataSegment> findSegments(String workingDirPathStr, boolean updateDescriptor) throws SegmentLoadingException { final Map<String, Pair<DataSegment, Long>> timestampedSegments = new HashMap<>(); final Path workingDirPath = new Path(workingDirPathStr); FileSystem fs; try {/*w ww.j a v a 2 s . co m*/ fs = workingDirPath.getFileSystem(config); log.info(fs.getScheme()); log.info("FileSystem URI:" + fs.getUri().toString()); if (!fs.exists(workingDirPath)) { throw new SegmentLoadingException("Working directory [%s] doesn't exist.", workingDirPath); } if (!fs.isDirectory(workingDirPath)) { throw new SegmentLoadingException("Working directory [%s] is not a directory!?", workingDirPath); } final RemoteIterator<LocatedFileStatus> it = fs.listFiles(workingDirPath, true); while (it.hasNext()) { final LocatedFileStatus locatedFileStatus = it.next(); final Path path = locatedFileStatus.getPath(); if (path.getName().endsWith("descriptor.json")) { // There are 3 supported path formats: // - hdfs://nn1/hdfs_base_directory/data_source_name/interval/version/shardNum/descriptor.json // - hdfs://nn1/hdfs_base_directory/data_source_name/interval/version/shardNum_descriptor.json // - hdfs://nn1/hdfs_base_directory/data_source_name/interval/version/shardNum_UUID_descriptor.json final String descriptorParts[] = path.getName().split("_"); Path indexZip = new Path(path.getParent(), "index.zip"); if (descriptorParts.length > 1) { Preconditions .checkState( descriptorParts.length <= 3 && org.apache.commons.lang.StringUtils.isNumeric(descriptorParts[0]) && "descriptor.json" .equals(descriptorParts[descriptorParts.length - 1]), "Unexpected descriptor filename format [%s]", path); indexZip = new Path(path.getParent(), StringUtils.format("%s_%sindex.zip", descriptorParts[0], descriptorParts.length == 2 ? "" : descriptorParts[1] + "_")); } if (fs.exists(indexZip)) { final DataSegment dataSegment = mapper.readValue(fs.open(path), DataSegment.class); log.info("Found segment [%s] located at [%s]", dataSegment.getIdentifier(), indexZip); final Map<String, Object> loadSpec = dataSegment.getLoadSpec(); final String pathWithoutScheme = indexZip.toUri().getPath(); if (!loadSpec.get("type").equals(HdfsStorageDruidModule.SCHEME) || !loadSpec.get("path").equals(pathWithoutScheme)) { loadSpec.put("type", HdfsStorageDruidModule.SCHEME); loadSpec.put("path", pathWithoutScheme); if (updateDescriptor) { log.info("Updating loadSpec in descriptor.json at [%s] with new path [%s]", path, pathWithoutScheme); mapper.writeValue(fs.create(path, true), dataSegment); } } DataSegmentFinder.putInMapRetainingNewest(timestampedSegments, dataSegment, locatedFileStatus.getModificationTime()); } else { throw new SegmentLoadingException( "index.zip didn't exist at [%s] while descripter.json exists!?", indexZip); } } } } catch (IOException e) { throw new SegmentLoadingException(e, "Problems interacting with filesystem[%s].", workingDirPath); } return timestampedSegments.values().stream().map(x -> x.lhs).collect(Collectors.toSet()); }
From source file:org.apache.druid.storage.hdfs.HdfsDataSegmentPuller.java
License:Apache License
FileUtils.FileCopyResult getSegmentFiles(final Path path, final File outDir) throws SegmentLoadingException { try {//from www . j av a2s . c o m org.apache.commons.io.FileUtils.forceMkdir(outDir); } catch (IOException e) { throw new SegmentLoadingException(e, ""); } try { final FileSystem fs = path.getFileSystem(config); if (fs.isDirectory(path)) { // -------- directory --------- try { return RetryUtils.retry(() -> { if (!fs.exists(path)) { throw new SegmentLoadingException("No files found at [%s]", path.toString()); } final RemoteIterator<LocatedFileStatus> children = fs.listFiles(path, false); final FileUtils.FileCopyResult result = new FileUtils.FileCopyResult(); while (children.hasNext()) { final LocatedFileStatus child = children.next(); final Path childPath = child.getPath(); final String fname = childPath.getName(); if (fs.isDirectory(childPath)) { log.warn("[%s] is a child directory, skipping", childPath.toString()); } else { final File outFile = new File(outDir, fname); try (final FSDataInputStream in = fs.open(childPath)) { NativeIO.chunkedCopy(in, outFile); } result.addFile(outFile); } } log.info("Copied %d bytes from [%s] to [%s]", result.size(), path.toString(), outDir.getAbsolutePath()); return result; }, shouldRetryPredicate(), DEFAULT_RETRY_COUNT); } catch (Exception e) { throw new RuntimeException(e); } } else if (CompressionUtils.isZip(path.getName())) { // -------- zip --------- final FileUtils.FileCopyResult result = CompressionUtils.unzip(new ByteSource() { @Override public InputStream openStream() throws IOException { return getInputStream(path); } }, outDir, shouldRetryPredicate(), false); log.info("Unzipped %d bytes from [%s] to [%s]", result.size(), path.toString(), outDir.getAbsolutePath()); return result; } else if (CompressionUtils.isGz(path.getName())) { // -------- gzip --------- final String fname = path.getName(); final File outFile = new File(outDir, CompressionUtils.getGzBaseName(fname)); final FileUtils.FileCopyResult result = CompressionUtils.gunzip(new ByteSource() { @Override public InputStream openStream() throws IOException { return getInputStream(path); } }, outFile); log.info("Gunzipped %d bytes from [%s] to [%s]", result.size(), path.toString(), outFile.getAbsolutePath()); return result; } else { throw new SegmentLoadingException("Do not know how to handle file type at [%s]", path.toString()); } } catch (IOException e) { throw new SegmentLoadingException(e, "Error loading [%s]", path.toString()); } }
From source file:org.apache.druid.storage.hdfs.HdfsFileTimestampVersionFinder.java
License:Apache License
/** * Returns the latest modified file at the uri of interest. * * @param uri Either a directory or a file on HDFS. If it is a file, the parent directory will be searched. * @param pattern A pattern matcher for file names in the directory of interest. Passing `null` results in matching any file in the directory. * * @return The URI of the file with the most recent modified timestamp. *///w ww .j a v a 2s .c o m @Override public URI getLatestVersion(final URI uri, final @Nullable Pattern pattern) { final Path path = new Path(uri); try { return RetryUtils.retry(() -> { final FileSystem fs = path.getFileSystem(config); if (!fs.exists(path)) { return null; } return mostRecentInDir(fs.isDirectory(path) ? path : path.getParent(), pattern); }, shouldRetryPredicate(), DEFAULT_RETRY_COUNT); } catch (Exception e) { throw new RuntimeException(e); } }
From source file:org.apache.druid.storage.hdfs.tasklog.HdfsTaskLogs.java
License:Apache License
@Override public void killOlderThan(long timestamp) throws IOException { Path taskLogDir = new Path(config.getDirectory()); FileSystem fs = taskLogDir.getFileSystem(hadoopConfig); if (fs.exists(taskLogDir)) { if (!fs.isDirectory(taskLogDir)) { throw new IOE("taskLogDir [%s] must be a directory.", taskLogDir); }//from w w w. j a va 2 s. c om RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(taskLogDir); while (iter.hasNext()) { LocatedFileStatus file = iter.next(); if (file.getModificationTime() < timestamp) { Path p = file.getPath(); log.info("Deleting hdfs task log [%s].", p.toUri().toString()); fs.delete(p, true); } if (Thread.currentThread().isInterrupted()) { throw new IOException( new InterruptedException("Thread interrupted. Couldn't delete all tasklogs.")); } } } }
From source file:org.apache.falcon.entity.parser.ClusterEntityParser.java
License:Apache License
private void createWorkingDirUnderStaging(FileSystem fs, Cluster cluster, Location stagingLocation) throws ValidationException { Path workingDirPath = new Path(stagingLocation.getPath(), ClusterHelper.WORKINGDIR); try {//from w ww.jav a 2s . com if (!fs.exists(workingDirPath)) { //Checking if the staging dir has the working dir to be created HadoopClientFactory.mkdirs(fs, workingDirPath, HadoopClientFactory.READ_EXECUTE_PERMISSION); } else { if (fs.isDirectory(workingDirPath)) { FsPermission workingPerms = fs.getFileStatus(workingDirPath).getPermission(); if (!workingPerms.equals(HadoopClientFactory.READ_EXECUTE_PERMISSION)) { //perms check throw new ValidationException( "Falcon needs subdir " + ClusterHelper.WORKINGDIR + " inside staging dir:" + stagingLocation.getPath() + " when staging location not specified with " + HadoopClientFactory.READ_EXECUTE_PERMISSION.toString() + " got " + workingPerms.toString()); } } else { throw new ValidationException("Falcon needs subdir " + ClusterHelper.WORKINGDIR + " inside staging dir:" + stagingLocation.getPath() + " when staging location not specified. Got a file at " + workingDirPath.toString()); } } } catch (IOException e) { throw new ValidationException("Unable to create path for " + workingDirPath.toString() + " with path: " + workingDirPath.toString() + " for cluster " + cluster.getName(), e); } }
From source file:org.apache.falcon.logging.LogProvider.java
License:Apache License
private Path findMaxRunIdPath(FileSystem fs, Path jobLogPath) throws IOException { // In case of multiple runs, dirs are sorted in increasing // order of runs. If runId is not specified, return the max runId (whose dir exists) Path maxRunIdPath = null;/*from w w w. j a v a2 s .c o m*/ for (FileStatus fileStatus : fs.globStatus(jobLogPath)) { if (fs.isDirectory(fileStatus.getPath())) { maxRunIdPath = fileStatus.getPath(); } } return maxRunIdPath; }
From source file:org.apache.gobblin.data.management.trash.Trash.java
License:Apache License
protected void ensureTrashLocationExists(FileSystem fs, Path trashLocation) throws IOException { if (fs.exists(trashLocation)) { if (!fs.isDirectory(trashLocation)) { throw new IOException(String.format("Trash location %s is not a directory.", trashLocation)); }//from w w w.j a va 2s . c o m if (!fs.exists(new Path(trashLocation, TRASH_IDENTIFIER_FILE))) { // If trash identifier file is not present, directory might have been created by user. // Add trash identifier file only if directory is empty. if (fs.listStatus(trashLocation).length > 0) { throw new IOException(String.format( "Trash directory %s exists, but it does not look like a trash directory. " + "File: %s missing and directory is not empty.", trashLocation, TRASH_IDENTIFIER_FILE)); } else if (!fs.createNewFile(new Path(trashLocation, TRASH_IDENTIFIER_FILE))) { throw new IOException(String.format("Failed to create file %s in existing trash directory %s.", TRASH_IDENTIFIER_FILE, trashLocation)); } } } else if (!(safeFsMkdir(fs, trashLocation.getParent(), ALL_PERM) && safeFsMkdir(fs, trashLocation, PERM) && fs.createNewFile(new Path(trashLocation, TRASH_IDENTIFIER_FILE)))) { // Failed to create directory or create trash identifier file. throw new IOException("Failed to create trash directory at " + trashLocation.toString()); } }