List of usage examples for org.apache.hadoop.fs FileSystem isFile
@Deprecated public boolean isFile(Path f) throws IOException
From source file:org.apache.oozie.service.AuthorizationService.java
License:Apache License
/** * Check if the user+group is authorized to use the specified application. <p> The check is done by checking the * file system permissions on the workflow application. * * @param user user name.// w w w. j ava 2 s . c om * @param group group name. * @param appPath application path. * @param fileName workflow or coordinator.xml * @param conf * @throws AuthorizationException thrown if the user is not authorized for the app. */ public void authorizeForApp(String user, String group, String appPath, String fileName, Configuration conf) throws AuthorizationException { try { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); URI uri = new Path(appPath).toUri(); Configuration fsConf = has.createJobConf(uri.getAuthority()); FileSystem fs = has.createFileSystem(user, uri, fsConf); Path path = new Path(appPath); try { if (!fs.exists(path)) { incrCounter(INSTR_FAILED_AUTH_COUNTER, 1); throw new AuthorizationException(ErrorCode.E0504, appPath); } if (conf.get(XOozieClient.IS_PROXY_SUBMISSION) == null) { // Only further check existence of job definition files for non proxy submission jobs; if (!fs.isFile(path)) { Path appXml = new Path(path, fileName); if (!fs.exists(appXml)) { incrCounter(INSTR_FAILED_AUTH_COUNTER, 1); throw new AuthorizationException(ErrorCode.E0505, appPath); } if (!fs.isFile(appXml)) { incrCounter(INSTR_FAILED_AUTH_COUNTER, 1); throw new AuthorizationException(ErrorCode.E0506, appPath); } fs.open(appXml).close(); } } } // TODO change this when stopping support of 0.18 to the new // Exception catch (org.apache.hadoop.fs.permission.AccessControlException ex) { incrCounter(INSTR_FAILED_AUTH_COUNTER, 1); throw new AuthorizationException(ErrorCode.E0507, appPath, ex.getMessage(), ex); } } catch (IOException ex) { incrCounter(INSTR_FAILED_AUTH_COUNTER, 1); throw new AuthorizationException(ErrorCode.E0501, ex.getMessage(), ex); } catch (HadoopAccessorException e) { throw new AuthorizationException(e); } }
From source file:org.apache.oozie.service.ShareLibService.java
License:Apache License
/** * Gets the path recursively.//w w w . j a v a 2 s .com * * @param fs the FileSystem * @param rootDir the root directory * @param listOfPaths the list of paths * @param shareLibKey the share lib key * @return the path recursively * @throws IOException Signals that an I/O exception has occurred. */ private void getPathRecursively(FileSystem fs, Path rootDir, List<Path> listOfPaths, String shareLibKey, Map<String, Map<Path, Configuration>> shareLibConfigMap) throws IOException { if (rootDir == null) { return; } try { if (fs.isFile(new Path(new URI(rootDir.toString()).getPath()))) { Path filePath = new Path(new URI(rootDir.toString()).getPath()); if (isFilePartOfConfList(rootDir)) { cachePropertyFile(filePath, shareLibKey, shareLibConfigMap); } listOfPaths.add(rootDir); return; } FileStatus[] status = fs.listStatus(rootDir); if (status == null) { LOG.info("Shared lib " + rootDir + " doesn't exist, not adding to cache"); return; } for (FileStatus file : status) { if (file.isDir()) { getPathRecursively(fs, file.getPath(), listOfPaths, shareLibKey, shareLibConfigMap); } else { if (isFilePartOfConfList(file.getPath())) { cachePropertyFile(file.getPath(), shareLibKey, shareLibConfigMap); } listOfPaths.add(file.getPath()); } } } catch (URISyntaxException e) { throw new IOException(e); } catch (JDOMException e) { throw new IOException(e); } }
From source file:org.apache.oozie.service.WorkflowAppService.java
License:Apache License
/** * Read workflow definition./*from w w w .j a v a 2 s. c o m*/ * * * @param appPath application path. * @param user user name. * @return workflow definition. * @throws WorkflowException thrown if the definition could not be read. */ protected String readDefinition(String appPath, String user, Configuration conf) throws WorkflowException { try { URI uri = new URI(appPath); HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); JobConf jobConf = has.createJobConf(uri.getAuthority()); FileSystem fs = has.createFileSystem(user, uri, jobConf); // app path could be a directory Path path = new Path(uri.getPath()); if (!fs.isFile(path)) { path = new Path(path, "workflow.xml"); } FileStatus fsStatus = fs.getFileStatus(path); if (fsStatus.getLen() > this.maxWFLength) { throw new WorkflowException(ErrorCode.E0736, fsStatus.getLen(), this.maxWFLength); } Reader reader = new InputStreamReader(fs.open(path)); StringWriter writer = new StringWriter(); IOUtils.copyCharStream(reader, writer); return writer.toString(); } catch (WorkflowException wfe) { throw wfe; } catch (IOException ex) { throw new WorkflowException(ErrorCode.E0710, ex.getMessage(), ex); } catch (URISyntaxException ex) { throw new WorkflowException(ErrorCode.E0711, appPath, ex.getMessage(), ex); } catch (HadoopAccessorException ex) { throw new WorkflowException(ex); } catch (Exception ex) { throw new WorkflowException(ErrorCode.E0710, ex.getMessage(), ex); } }
From source file:org.apache.oozie.service.WorkflowAppService.java
License:Apache License
/** * Create proto configuration. <p> The proto configuration includes the user,group and the paths which need to be * added to distributed cache. These paths include .jar,.so and the resource file paths. * * @param jobConf job configuration./*from w w w . ja va 2s . c om*/ * @param isWorkflowJob indicates if the job is a workflow job or not. * @return proto configuration. * @throws WorkflowException thrown if the proto action configuration could not be created. */ public XConfiguration createProtoActionConf(Configuration jobConf, boolean isWorkflowJob) throws WorkflowException { try { HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); URI uri = new URI(jobConf.get(OozieClient.APP_PATH)); Configuration conf = has.createJobConf(uri.getAuthority()); XConfiguration protoConf = new XConfiguration(); String user = jobConf.get(OozieClient.USER_NAME); conf.set(OozieClient.USER_NAME, user); protoConf.set(OozieClient.USER_NAME, user); FileSystem fs = has.createFileSystem(user, uri, conf); Path appPath = new Path(uri); XLog.getLog(getClass()).debug("jobConf.libPath = " + jobConf.get(OozieClient.LIBPATH)); XLog.getLog(getClass()).debug("jobConf.appPath = " + appPath); Collection<String> filePaths; if (isWorkflowJob) { // app path could be a directory Path path = new Path(uri.getPath()); if (!fs.isFile(path)) { filePaths = getLibFiles(fs, new Path(appPath + "/lib")); } else { filePaths = getLibFiles(fs, new Path(appPath.getParent(), "lib")); } } else { filePaths = new LinkedHashSet<String>(); } String[] libPaths = jobConf.getStrings(OozieClient.LIBPATH); if (libPaths != null && libPaths.length > 0) { for (int i = 0; i < libPaths.length; i++) { if (libPaths[i].trim().length() > 0) { Path libPath = new Path(libPaths[i].trim()); Collection<String> libFilePaths = getLibFiles(fs, libPath); filePaths.addAll(libFilePaths); } } } // Check if a subworkflow should inherit the libs from the parent WF // OOZIE_WF_SUBWORKFLOW_CLASSPATH_INHERITANCE has priority over OOZIE_SUBWORKFLOW_CLASSPATH_INHERITANCE from oozie-site // If OOZIE_WF_SUBWORKFLOW_CLASSPATH_INHERITANCE isn't specified, we use OOZIE_SUBWORKFLOW_CLASSPATH_INHERITANCE if (jobConf.getBoolean(OOZIE_WF_SUBWORKFLOW_CLASSPATH_INHERITANCE, oozieSubWfCPInheritance)) { // Keep any libs from a parent workflow that might already be in APP_LIB_PATH_LIST and also remove duplicates String[] parentFilePaths = jobConf.getStrings(APP_LIB_PATH_LIST); if (parentFilePaths != null && parentFilePaths.length > 0) { String[] filePathsNames = filePaths.toArray(new String[filePaths.size()]); for (int i = 0; i < filePathsNames.length; i++) { Path p = new Path(filePathsNames[i]); filePathsNames[i] = p.getName(); } Arrays.sort(filePathsNames); List<String> nonDuplicateParentFilePaths = new ArrayList<String>(); for (String parentFilePath : parentFilePaths) { Path p = new Path(parentFilePath); if (Arrays.binarySearch(filePathsNames, p.getName()) < 0) { nonDuplicateParentFilePaths.add(parentFilePath); } } filePaths.addAll(nonDuplicateParentFilePaths); } } protoConf.setStrings(APP_LIB_PATH_LIST, filePaths.toArray(new String[filePaths.size()])); //Add all properties start with 'oozie.' for (Map.Entry<String, String> entry : jobConf) { if (entry.getKey().startsWith("oozie.")) { String name = entry.getKey(); String value = entry.getValue(); // if property already exists, should not overwrite if (protoConf.get(name) == null) { protoConf.set(name, value); } } } return protoConf; } catch (IOException ex) { throw new WorkflowException(ErrorCode.E0712, jobConf.get(OozieClient.APP_PATH), ex.getMessage(), ex); } catch (URISyntaxException ex) { throw new WorkflowException(ErrorCode.E0711, jobConf.get(OozieClient.APP_PATH), ex.getMessage(), ex); } catch (HadoopAccessorException ex) { throw new WorkflowException(ex); } catch (Exception ex) { throw new WorkflowException(ErrorCode.E0712, jobConf.get(OozieClient.APP_PATH), ex.getMessage(), ex); } }
From source file:org.apache.pig.builtin.TestAvroStorage.java
License:Apache License
private void verifyResults(String outPath, String expectedOutpath, String expectedCodec) throws IOException { FileSystem fs = FileSystem.getLocal(new Configuration()); /* read in expected results*/ Set<GenericData.Record> expected = getExpected(expectedOutpath); /* read in output results and compare */ Path output = new Path(outPath); assertTrue("Output dir does not exists!", fs.exists(output) && fs.getFileStatus(output).isDir()); Path[] paths = FileUtil.stat2Paths(fs.listStatus(output, hiddenPathFilter)); assertTrue("Split field dirs not found!", paths != null); for (Path path : paths) { Path[] files = FileUtil.stat2Paths(fs.listStatus(path, hiddenPathFilter)); assertTrue("No files found for path: " + path.toUri().getPath(), files != null); for (Path filePath : files) { assertTrue("This shouldn't be a directory", fs.isFile(filePath)); GenericDatumReader<GenericData.Record> reader = new GenericDatumReader<GenericData.Record>(); DataFileStream<GenericData.Record> in = new DataFileStream<GenericData.Record>(fs.open(filePath), reader);/* w ww .j a v a 2 s . c om*/ assertEquals("codec", expectedCodec, in.getMetaString("avro.codec")); int count = 0; while (in.hasNext()) { GenericData.Record obj = in.next(); assertTrue( "Avro result object found that's not expected: Found " + (obj != null ? obj.getSchema() : "null") + ", " + obj.toString() + "\nExpected " + (expected != null ? expected.toString() : "null") + "\n", expected.contains(obj)); count++; } in.close(); assertEquals(expected.size(), count); } } }
From source file:org.apache.pig.builtin.TestAvroStorage.java
License:Apache License
private Set<GenericData.Record> getExpected(String pathstr) throws IOException { Set<GenericData.Record> ret = new TreeSet<GenericData.Record>(new Comparator<GenericData.Record>() { @Override/*from w ww .j ava2s .c o m*/ public int compare(Record o1, Record o2) { return o1.toString().compareTo(o2.toString()); } }); FileSystem fs = FileSystem.getLocal(new Configuration()); /* read in output results and compare */ Path output = new Path(pathstr); assertTrue("Expected output does not exists!", fs.exists(output)); Path[] paths = FileUtil.stat2Paths(fs.listStatus(output, hiddenPathFilter)); assertTrue("Split field dirs not found!", paths != null); for (Path path : paths) { Path[] files = FileUtil.stat2Paths(fs.listStatus(path, hiddenPathFilter)); assertTrue("No files found for path: " + path.toUri().getPath(), files != null); for (Path filePath : files) { assertTrue("This shouldn't be a directory", fs.isFile(filePath)); GenericDatumReader<GenericData.Record> reader = new GenericDatumReader<GenericData.Record>(); DataFileStream<GenericData.Record> in = new DataFileStream<GenericData.Record>(fs.open(filePath), reader); while (in.hasNext()) { GenericData.Record obj = in.next(); ret.add(obj); } in.close(); } } return ret; }
From source file:org.apache.pig.impl.util.Utils.java
License:Apache License
/** * Finds a valid path for a file from a FileStatus object. * @param fileStatus FileStatus object corresponding to a file, * or a directory./* w ww.ja v a 2 s. c o m*/ * @param fileSystem FileSystem in with the file should be found * @return The first file found * @throws IOException */ public static Path depthFirstSearchForFile(final FileStatus fileStatus, final FileSystem fileSystem) throws IOException { if (fileSystem.isFile(fileStatus.getPath())) { return fileStatus.getPath(); } else { return depthFirstSearchForFile(fileSystem.listStatus(fileStatus.getPath(), VISIBLE_FILES), fileSystem); } }
From source file:org.apache.pig.piggybank.storage.partition.PathPartitioner.java
License:Apache License
/** * Searches for the key=value pairs in the path pointer by the location * parameter.//from w ww. j a v a 2 s . co m * * @param location * String root path in hdsf e.g. /user/hive/warehouse or * /logs/repo * @param conf * Configuration * @return Set of String. The order is maintained as per the directory tree. * i.e. if /logs/repo/year=2010/month=2010 exists the first item in * the set will be year and the second month. * @throws IOException */ public Set<String> getPartitionKeys(String location, Configuration conf) throws IOException { // find the hive type partition key=value pairs from the path. // first parse the string alone. Path path = new Path(location); FileSystem fs = path.getFileSystem(conf); FileStatus[] fileStatusArr = null; // use LinkedHashSet because order is important here. Set<String> partitionKeys = new LinkedHashSet<String>(); parseAndPutKeyValue(location, partitionKeys); while (!((fileStatusArr = fs.listStatus(path)) == null || fs.isFile(path) || fileStatusArr.length == 0)) { for (FileStatus fileStatus : fileStatusArr) { path = fileStatus.getPath(); // ignore hidden directories if (fileStatus.getPath().getName().startsWith("_") || !fileStatus.isDir()) continue; parseAndPutKeyValue(path.getName(), partitionKeys); // at the first directory found stop the for loop after parsing // for key value pairs break; } } return partitionKeys; }
From source file:org.apache.pig.piggybank.test.storage.avro.TestAvroStorage.java
License:Apache License
private void verifyResults(String outPath, String expectedOutpath, String expectedCodec) throws IOException { FileSystem fs = FileSystem.getLocal(new Configuration()); /* read in expected results*/ Set<Object> expected = getExpected(expectedOutpath); /* read in output results and compare */ Path output = new Path(outPath); assertTrue("Output dir does not exists!", fs.exists(output) && fs.getFileStatus(output).isDir()); Path[] paths = FileUtil.stat2Paths(fs.listStatus(output, hiddenPathFilter)); assertTrue("Split field dirs not found!", paths != null); for (Path path : paths) { Path[] files = FileUtil.stat2Paths(fs.listStatus(path, hiddenPathFilter)); assertTrue("No files found for path: " + path.toUri().getPath(), files != null); for (Path filePath : files) { assertTrue("This shouldn't be a directory", fs.isFile(filePath)); GenericDatumReader<Object> reader = new GenericDatumReader<Object>(); DataFileStream<Object> in = new DataFileStream<Object>(fs.open(filePath), reader); assertEquals("codec", expectedCodec, in.getMetaString("avro.codec")); int count = 0; while (in.hasNext()) { Object obj = in.next(); //System.out.println("obj = " + (GenericData.Array<Float>)obj); assertTrue("Avro result object found that's not expected: " + obj, expected.contains(obj)); count++;//from w w w. j a v a2 s . co m } in.close(); assertEquals(expected.size(), count); } } }
From source file:org.apache.pig.piggybank.test.storage.avro.TestAvroStorage.java
License:Apache License
private Set<Object> getExpected(String pathstr) throws IOException { Set<Object> ret = new HashSet<Object>(); FileSystem fs = FileSystem.getLocal(new Configuration()); /* read in output results and compare */ Path output = new Path(pathstr); assertTrue("Expected output does not exists!", fs.exists(output)); Path[] paths = FileUtil.stat2Paths(fs.listStatus(output, hiddenPathFilter)); assertTrue("Split field dirs not found!", paths != null); for (Path path : paths) { Path[] files = FileUtil.stat2Paths(fs.listStatus(path, hiddenPathFilter)); assertTrue("No files found for path: " + path.toUri().getPath(), files != null); for (Path filePath : files) { assertTrue("This shouldn't be a directory", fs.isFile(filePath)); GenericDatumReader<Object> reader = new GenericDatumReader<Object>(); DataFileStream<Object> in = new DataFileStream<Object>(fs.open(filePath), reader); while (in.hasNext()) { Object obj = in.next(); ret.add(obj);/* ww w . j ava 2 s .c o m*/ } in.close(); } } return ret; }