List of usage examples for org.apache.hadoop.fs FileSystem mkdirs
public static boolean mkdirs(FileSystem fs, Path dir, FsPermission permission) throws IOException
From source file:alluxio.underfs.hdfs.HdfsUnderFileSystem.java
License:Apache License
@Override public boolean mkdirs(String path, MkdirsOptions options) throws IOException { IOException te = null;/* w ww. j av a 2 s .c o m*/ RetryPolicy retryPolicy = new CountingRetry(MAX_TRY); while (retryPolicy.attemptRetry()) { try { Path hdfsPath = new Path(path); if (mFileSystem.exists(hdfsPath)) { LOG.debug("Trying to create existing directory at {}", path); return false; } // Create directories one by one with explicit permissions to ensure no umask is applied, // using mkdirs will apply the permission only to the last directory Stack<Path> dirsToMake = new Stack<>(); dirsToMake.push(hdfsPath); Path parent = hdfsPath.getParent(); while (!mFileSystem.exists(parent)) { dirsToMake.push(parent); parent = parent.getParent(); } while (!dirsToMake.empty()) { if (!FileSystem.mkdirs(mFileSystem, dirsToMake.pop(), new FsPermission(options.getPermission().getMode().toShort()))) { return false; } } return true; } catch (IOException e) { LOG.error("{} try to make directory for {} : {}", retryPolicy.getRetryCount(), path, e.getMessage(), e); te = e; } } throw te; }
From source file:audr.text.utils.FileUtils.java
License:Open Source License
/** * create index dirs for each text category * @param root/* ww w .j a va 2s . c o m*/ * @throws IOException */ public static void makeIndexDirs() throws IOException { FileSystem fs = FileSystem.get(new Configuration()); for (int i = 0; i < TextCategoryFields.TEXT_CATEGOTIES_ENUM.length; ++i) { String oriDir = Constants.INPUT_PATH.replace("%Category%", TextCategoryFields.TEXT_CATEGOTIES_ENUM[i]); String lfDir = Constants.INPUT_PATH_LF.replace("%Category%", TextCategoryFields.TEXT_CATEGOTIES_ENUM[i]); FileSystem.mkdirs(fs, new Path(oriDir), FsPermission.getDefault()); FileSystem.mkdirs(fs, new Path(lfDir), FsPermission.getDefault()); for (int j = 0; j < Constants.INDEX_SHARDS.length; ++j) { String indexDir = Constants.INDEX_SHARDS[j].replace("%Category%", TextCategoryFields.TEXT_CATEGOTIES_ENUM[i]); FileSystem.mkdirs(fs, new Path(indexDir), FsPermission.getDefault()); } } }
From source file:com.chinamobile.bcbsp.client.BSPJobClient.java
License:Apache License
/** * Submit a new job to run./* w ww.j av a2 s . com*/ * @param job BSPJob * @return Review comments: (1)The content of submitJobDir is decided by the * client. I think it is dangerous because two different clients maybe * generate the same submitJobDir. Review time: 2011-11-30; Reviewer: * Hongxu Zhang. Fix log: (1)In order to avoid the conflict, I use the * jobId to generate the submitJobDir. Because the jobId is unique so * this problem can be solved. Fix time: 2011-12-04; Programmer: * Zhigang Wang. Review comments: (2)There, the client must submit * relative information about the job. There maybe some exceptions * during this process. When exceptions occur, this job should not be * executed and the relative submitJobDir must be cleanup. Review * time: 2011-12-04; Reviewer: Hongxu Zhang. Fix log: (2)The process * of submiting files has been surrounded by try-catch. The * submitJobDir will be cleanup in the catch process. Fix time: * 2011-12-04; Programmer: Zhigang Wang. */ public RunningJob submitJobInternal(BSPJob job) { BSPJobID jobId = null; Path submitJobDir = null; try { jobId = jobSubmitClient.getNewJobId(); submitJobDir = new Path(getSystemDir(), "submit_" + jobId.toString()); Path submitJarFile = null; LOG.info("debug: job type is " + job.getJobType()); if (Constants.USER_BC_BSP_JOB_TYPE_C.equals(job.getJobType())) { submitJarFile = new Path(submitJobDir, "jobC"); LOG.info("debug:" + submitJarFile.toString()); } else { LOG.info("debug: before submitJarFile = new " + "Path(submitJobDir,job.jar);"); submitJarFile = new Path(submitJobDir, "job.jar"); LOG.info("debug:" + submitJarFile.toString()); } Path submitJobFile = new Path(submitJobDir, "job.xml"); Path submitSplitFile = new Path(submitJobDir, "job.split"); // set this user's id in job configuration, so later job files can // be accessed using this user's id UnixUserGroupInformation ugi = getUGI(job.getConf()); // Create a number of filenames in the BSPController's fs namespace FileSystem files = getFs(); files.delete(submitJobDir, true); submitJobDir = files.makeQualified(submitJobDir); submitJobDir = new Path(submitJobDir.toUri().getPath()); BSPFsPermission bspSysPerms = new BSPFspermissionImpl(2); FileSystem.mkdirs(files, submitJobDir, bspSysPerms.getFp()); files.mkdirs(submitJobDir); short replication = (short) job.getInt("bsp.submit.replication", 10); String originalJarPath = null; LOG.info("debug: job type is " + job.getJobType()); if (Constants.USER_BC_BSP_JOB_TYPE_C.equals(job.getJobType())) { LOG.info("debug: originalJarPath = job.getJobExe();" + job.getJobExe()); originalJarPath = job.getJobExe(); LOG.info("debug:" + submitJarFile.toString()); job.setJobExe(submitJarFile.toString()); } else { LOG.info("debug: jar"); originalJarPath = job.getJar(); job.setJar(submitJarFile.toString()); } if (originalJarPath != null) { // copy jar to BSPController's fs // use jar name if job is not named. if ("".equals(job.getJobName())) { job.setJobName(new Path(originalJarPath).getName()); } // job.setJar(submitJarFile.toString()); fs.copyFromLocalFile(new Path(originalJarPath), submitJarFile); fs.setReplication(submitJarFile, replication); fs.setPermission(submitJarFile, new BSPFspermissionImpl(0).getFp()); } else { LOG.warn("No job jar file set. User classes may not be found. " + "See BSPJob#setJar(String) or check Your jar file."); } // Set the user's name and working directory job.setUser(ugi.getUserName()); if (ugi.getGroupNames().length > 0) { job.set("group.name", ugi.getGroupNames()[0]); } if (new BSPHdfsImpl().getWorkingDirectory() == null) { job.setWorkingDirectory(fs.getWorkingDirectory()); } int maxClusterStaffs = jobSubmitClient.getClusterStatus(false).getMaxClusterStaffs(); if (job.getNumPartition() == 0) { job.setNumPartition(maxClusterStaffs); } if (job.getNumPartition() > maxClusterStaffs) { job.setNumPartition(maxClusterStaffs); } job.setNumBspStaff(job.getNumPartition()); int splitNum = 0; splitNum = writeSplits(job, submitSplitFile); if (splitNum > job.getNumPartition() && splitNum <= maxClusterStaffs) { job.setNumPartition(splitNum); job.setNumBspStaff(job.getNumPartition()); } if (splitNum > maxClusterStaffs) { LOG.error("Sorry, the number of files is more than maxClusterStaffs:" + maxClusterStaffs); throw new IOException("Could not launch job"); } job.set(Constants.USER_BC_BSP_JOB_SPLIT_FILE, submitSplitFile.toString()); LOG.info("[Max Staff Number] " + maxClusterStaffs); LOG.info("The number of splits for the job is: " + splitNum); LOG.info("The number of staffs for the job is: " + job.getNumBspStaff()); BSPFSDataOutputStream bspout = new BSPFSDataOutputStreamImpl(fs, submitJobFile, new BSPFspermissionImpl(0).getFp()); try { job.writeXml(bspout.getOut()); } finally { bspout.close(); } // Now, actually submit the job (using the submit name) JobStatus status = jobSubmitClient.submitJob(jobId, submitJobFile.toString()); if (status != null) { return new NetworkedJob(status); } else { throw new IOException("Could not launch job"); } } catch (FileNotFoundException fnfE) { LOG.error("Exception has been catched in BSPJobClient--submitJobInternal !", fnfE); Fault f = new Fault(Fault.Type.SYSTEMSERVICE, Fault.Level.INDETERMINATE, "null", fnfE.toString()); jobSubmitClient.recordFault(f); jobSubmitClient.recovery(jobId); try { FileSystem files = getFs(); files.delete(submitJobDir, true); } catch (IOException e) { //LOG.error("Failed to cleanup the submitJobDir:" + submitJobDir); throw new RuntimeException("Failed to cleanup the submitJobDir", e); } return null; } catch (ClassNotFoundException cnfE) { LOG.error("Exception has been catched in BSPJobClient--submitJobInternal !", cnfE); Fault f = new Fault(Fault.Type.SYSTEMSERVICE, Fault.Level.WARNING, "null", cnfE.toString()); jobSubmitClient.recordFault(f); jobSubmitClient.recovery(jobId); try { FileSystem files = getFs(); files.delete(submitJobDir, true); } catch (IOException e) { //LOG.error("Failed to cleanup the submitJobDir:" + submitJobDir); throw new RuntimeException("Failed to cleanup the submitJobDir", e); } return null; } catch (InterruptedException iE) { LOG.error("Exception has been catched in BSPJobClient--submitJobInternal !", iE); Fault f = new Fault(Fault.Type.SYSTEMSERVICE, Fault.Level.CRITICAL, "null", iE.toString()); jobSubmitClient.recordFault(f); jobSubmitClient.recovery(jobId); try { FileSystem files = getFs(); files.delete(submitJobDir, true); } catch (IOException e) { //LOG.error("Failed to cleanup the submitJobDir:" + submitJobDir); throw new RuntimeException("Failed to cleanup the submitJobDir", e); } return null; } catch (Exception ioE) { LOG.error("Exception has been catched in BSPJobClient--submitJobInternal !", ioE); Fault f = new Fault(Fault.Type.DISK, Fault.Level.CRITICAL, "null", ioE.toString()); jobSubmitClient.recordFault(f); jobSubmitClient.recovery(jobId); try { FileSystem files = getFs(); files.delete(submitJobDir, true); } catch (IOException e) { //LOG.error("Failed to cleanup the submitJobDir:" + submitJobDir); throw new RuntimeException("Failed to cleanup the submitJobDir", e); } return null; } }
From source file:com.kadwa.hadoop.DistExec.java
License:Open Source License
/** * Initialize ExecFilesMapper specific job-configuration. * * @param conf : The dfs/mapred configuration. * @param jobConf : The handle to the jobConf object to be initialized. * @param args Arguments//from w w w .j a va 2 s . c o m * @return true if it is necessary to launch a job. */ private static boolean setup(Configuration conf, JobConf jobConf, final Arguments args) throws IOException { jobConf.set(DST_DIR_LABEL, args.dst.toUri().toString()); jobConf.set(EXEC_CMD_LABEL, args.execCmd); //set boolean values jobConf.setBoolean(Options.REDIRECT_ERROR_TO_OUT.propertyname, args.flags.contains(Options.REDIRECT_ERROR_TO_OUT)); final String randomId = getRandomId(); JobClient jClient = new JobClient(jobConf); Path stagingArea; try { stagingArea = JobSubmissionFiles.getStagingDir(jClient, conf); } catch (InterruptedException e) { throw new IOException(e); } Path jobDirectory = new Path(stagingArea + NAME + "_" + randomId); FsPermission mapredSysPerms = new FsPermission(JobSubmissionFiles.JOB_DIR_PERMISSION); FileSystem.mkdirs(FileSystem.get(jobDirectory.toUri(), conf), jobDirectory, mapredSysPerms); jobConf.set(JOB_DIR_LABEL, jobDirectory.toString()); FileSystem dstfs = args.dst.getFileSystem(conf); // get tokens for all the required FileSystems.. TokenCache.obtainTokensForNamenodes(jobConf.getCredentials(), new Path[] { args.dst }, conf); boolean dstExists = dstfs.exists(args.dst); boolean dstIsDir = false; if (dstExists) { dstIsDir = dstfs.getFileStatus(args.dst).isDir(); } // default logPath Path logPath = args.log; if (logPath == null) { String filename = "_" + NAME + "_logs_" + randomId; if (!dstExists || !dstIsDir) { Path parent = args.dst.getParent(); if (!dstfs.exists(parent)) { dstfs.mkdirs(parent); } logPath = new Path(parent, filename); } else { logPath = new Path(args.dst, filename); } } FileOutputFormat.setOutputPath(jobConf, logPath); // create src list, dst list FileSystem jobfs = jobDirectory.getFileSystem(jobConf); Path srcfilelist = new Path(jobDirectory, "_" + NAME + "_src_files"); jobConf.set(SRC_LIST_LABEL, srcfilelist.toString()); SequenceFile.Writer src_writer = SequenceFile.createWriter(jobfs, jobConf, srcfilelist, LongWritable.class, FilePair.class, SequenceFile.CompressionType.NONE); Path dstfilelist = new Path(jobDirectory, "_" + NAME + "_dst_files"); SequenceFile.Writer dst_writer = SequenceFile.createWriter(jobfs, jobConf, dstfilelist, Text.class, Text.class, SequenceFile.CompressionType.NONE); Path dstdirlist = new Path(jobDirectory, "_" + NAME + "_dst_dirs"); jobConf.set(DST_DIR_LIST_LABEL, dstdirlist.toString()); SequenceFile.Writer dir_writer = SequenceFile.createWriter(jobfs, jobConf, dstdirlist, Text.class, FilePair.class, SequenceFile.CompressionType.NONE); // handle the case where the destination directory doesn't exist // and we've only a single src directory. final boolean special = (args.srcs.size() == 1 && !dstExists); int srcCount = 0, cnsyncf = 0, dirsyn = 0; long fileCount = 0L, byteCount = 0L, cbsyncs = 0L; try { for (Iterator<Path> srcItr = args.srcs.iterator(); srcItr.hasNext();) { final Path src = srcItr.next(); FileSystem srcfs = src.getFileSystem(conf); FileStatus srcfilestat = srcfs.getFileStatus(src); Path root = special && srcfilestat.isDir() ? src : src.getParent(); if (srcfilestat.isDir()) { ++srcCount; } Stack<FileStatus> pathstack = new Stack<FileStatus>(); for (pathstack.push(srcfilestat); !pathstack.empty();) { FileStatus cur = pathstack.pop(); FileStatus[] children = srcfs.listStatus(cur.getPath()); for (int i = 0; i < children.length; i++) { boolean skipfile = false; final FileStatus child = children[i]; final String dst = makeRelative(root, child.getPath()); ++srcCount; if (child.isDir()) { pathstack.push(child); } else { if (!skipfile) { ++fileCount; byteCount += child.getLen(); if (LOG.isTraceEnabled()) { LOG.trace("adding file " + child.getPath()); } ++cnsyncf; cbsyncs += child.getLen(); if (cnsyncf > SYNC_FILE_MAX || cbsyncs > BYTES_PER_MAP) { src_writer.sync(); dst_writer.sync(); cnsyncf = 0; cbsyncs = 0L; } } } if (!skipfile) { src_writer.append(new LongWritable(child.isDir() ? 0 : child.getLen()), new FilePair(child, dst)); } dst_writer.append(new Text(dst), new Text(child.getPath().toString())); } if (cur.isDir()) { String dst = makeRelative(root, cur.getPath()); dir_writer.append(new Text(dst), new FilePair(cur, dst)); if (++dirsyn > SYNC_FILE_MAX) { dirsyn = 0; dir_writer.sync(); } } } } } finally { checkAndClose(src_writer); checkAndClose(dst_writer); checkAndClose(dir_writer); } FileStatus dststatus = null; try { dststatus = dstfs.getFileStatus(args.dst); } catch (FileNotFoundException fnfe) { LOG.info(args.dst + " does not exist."); } // create dest path dir if copying > 1 file if (dststatus == null) { if (srcCount > 1 && !dstfs.mkdirs(args.dst)) { throw new IOException("Failed to create" + args.dst); } } final Path sorted = new Path(jobDirectory, "_" + NAME + "_sorted"); checkDuplication(jobfs, dstfilelist, sorted, conf); Path tmpDir = new Path( (dstExists && !dstIsDir) || (!dstExists && srcCount == 1) ? args.dst.getParent() : args.dst, "_" + NAME + "_tmp_" + randomId); jobConf.set(TMP_DIR_LABEL, tmpDir.toUri().toString()); LOG.info("sourcePathsCount=" + srcCount); LOG.info("filesToExecCount=" + fileCount); LOG.info("bytesToExecCount=" + StringUtils.humanReadableInt(byteCount)); jobConf.setInt(SRC_COUNT_LABEL, srcCount); jobConf.setLong(TOTAL_SIZE_LABEL, byteCount); setMapCount(fileCount, jobConf); return fileCount > 0; }
From source file:com.uber.hoodie.utilities.HiveIncrementalPuller.java
License:Apache License
private boolean ensureTempPathExists(FileSystem fs, String lastCommitTime) throws IOException { Path targetBaseDirPath = new Path(config.hoodieTmpDir, config.targetTable + "__" + config.sourceTable); if (!fs.exists(targetBaseDirPath)) { log.info("Creating " + targetBaseDirPath + " with permission drwxrwxrwx"); boolean result = FileSystem.mkdirs(fs, targetBaseDirPath, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); if (!result) { throw new HoodieException( "Could not create " + targetBaseDirPath + " with the required permissions"); }/*from w w w . j ava 2s. c om*/ } Path targetPath = new Path(targetBaseDirPath, lastCommitTime); if (fs.exists(targetPath)) { boolean result = fs.delete(targetPath, true); if (!result) { throw new HoodieException("Could not delete existing " + targetPath); } } log.info("Creating " + targetPath + " with permission drwxrwxrwx"); return FileSystem.mkdirs(fs, targetBaseDirPath, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); }
From source file:net.sf.jfilesync.plugins.net.items.THdfs_plugin.java
License:Apache License
@Override public void mkdir(String dir) throws IOException { boolean commandOK = FileSystem.mkdirs(fs, new Path(dir), FsPermission.getDefault()); if (!commandOK) { LOGGER.warning("Failed to make directory " + dir); }//from w ww . j a v a 2 s . c o m }