List of usage examples for org.apache.hadoop.fs FileSystem createNewFile
public boolean createNewFile(Path f) throws IOException
From source file:com.uber.hoodie.common.model.HoodieTestUtils.java
License:Apache License
public static final String createNewLogFile(FileSystem fs, String basePath, String partitionPath, String commitTime, String fileID, Optional<Integer> version) throws IOException { String folderPath = basePath + "/" + partitionPath + "/"; boolean makeDir = fs.mkdirs(new Path(folderPath)); if (!makeDir) { throw new IOException("cannot create directory for path " + folderPath); }/*from w w w.jav a 2s .co m*/ boolean createFile = fs.createNewFile(new Path(folderPath + FSUtils.makeLogFileName(fileID, ".log", commitTime, version.orElse(DEFAULT_LOG_VERSION), HoodieLogFormat.UNKNOWN_WRITE_TOKEN))); if (!createFile) { throw new IOException( StringUtils.format("cannot create data file for commit %s and fileId %s", commitTime, fileID)); } return fileID; }
From source file:com.uber.hoodie.common.model.HoodieTestUtils.java
License:Apache License
public static final void createCompactionCommitFiles(FileSystem fs, String basePath, String... commitTimes) throws IOException { for (String commitTime : commitTimes) { boolean createFile = fs.createNewFile(new Path(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + HoodieTimeline.makeCommitFileName(commitTime))); if (!createFile) { throw new IOException("cannot create commit file for commit " + commitTime); }/* w ww . java2s.c om*/ } }
From source file:com.uber.hoodie.utilities.HoodieSnapshotCopier.java
License:Apache License
public void snapshot(JavaSparkContext jsc, String baseDir, final String outputDir, final boolean shouldAssumeDatePartitioning) throws IOException { FileSystem fs = FSUtils.getFs(baseDir, jsc.hadoopConfiguration()); final SerializableConfiguration serConf = new SerializableConfiguration(jsc.hadoopConfiguration()); final HoodieTableMetaClient tableMetadata = new HoodieTableMetaClient(fs.getConf(), baseDir); final TableFileSystemView.ReadOptimizedView fsView = new HoodieTableFileSystemView(tableMetadata, tableMetadata.getActiveTimeline().getCommitsTimeline().filterCompletedInstants()); // Get the latest commit Optional<HoodieInstant> latestCommit = tableMetadata.getActiveTimeline().getCommitsTimeline() .filterCompletedInstants().lastInstant(); if (!latestCommit.isPresent()) { logger.warn("No commits present. Nothing to snapshot"); return;/* w w w .j a v a 2 s .co m*/ } final String latestCommitTimestamp = latestCommit.get().getTimestamp(); logger.info(String.format("Starting to snapshot latest version files which are also no-late-than %s.", latestCommitTimestamp)); List<String> partitions = FSUtils.getAllPartitionPaths(fs, baseDir, shouldAssumeDatePartitioning); if (partitions.size() > 0) { logger.info(String.format("The job needs to copy %d partitions.", partitions.size())); // Make sure the output directory is empty Path outputPath = new Path(outputDir); if (fs.exists(outputPath)) { logger.warn( String.format("The output path %s targetBasePath already exists, deleting", outputPath)); fs.delete(new Path(outputDir), true); } jsc.parallelize(partitions, partitions.size()).flatMap(partition -> { // Only take latest version files <= latestCommit. FileSystem fs1 = FSUtils.getFs(baseDir, serConf.get()); List<Tuple2<String, String>> filePaths = new ArrayList<>(); Stream<HoodieDataFile> dataFiles = fsView.getLatestDataFilesBeforeOrOn(partition, latestCommitTimestamp); dataFiles.forEach( hoodieDataFile -> filePaths.add(new Tuple2<>(partition, hoodieDataFile.getPath()))); // also need to copy over partition metadata Path partitionMetaFile = new Path(new Path(baseDir, partition), HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE); if (fs1.exists(partitionMetaFile)) { filePaths.add(new Tuple2<>(partition, partitionMetaFile.toString())); } return filePaths.iterator(); }).foreach(tuple -> { String partition = tuple._1(); Path sourceFilePath = new Path(tuple._2()); Path toPartitionPath = new Path(outputDir, partition); FileSystem ifs = FSUtils.getFs(baseDir, serConf.get()); if (!ifs.exists(toPartitionPath)) { ifs.mkdirs(toPartitionPath); } FileUtil.copy(ifs, sourceFilePath, ifs, new Path(toPartitionPath, sourceFilePath.getName()), false, ifs.getConf()); }); // Also copy the .commit files logger.info(String.format("Copying .commit files which are no-late-than %s.", latestCommitTimestamp)); FileStatus[] commitFilesToCopy = fs.listStatus( new Path(baseDir + "/" + HoodieTableMetaClient.METAFOLDER_NAME), (commitFilePath) -> { if (commitFilePath.getName().equals(HoodieTableConfig.HOODIE_PROPERTIES_FILE)) { return true; } else { String commitTime = FSUtils.getCommitFromCommitFile(commitFilePath.getName()); return HoodieTimeline.compareTimestamps(commitTime, latestCommitTimestamp, HoodieTimeline.LESSER_OR_EQUAL); } }); for (FileStatus commitStatus : commitFilesToCopy) { Path targetFilePath = new Path(outputDir + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + commitStatus.getPath().getName()); if (!fs.exists(targetFilePath.getParent())) { fs.mkdirs(targetFilePath.getParent()); } if (fs.exists(targetFilePath)) { logger.error(String.format("The target output commit file (%s targetBasePath) already exists.", targetFilePath)); } FileUtil.copy(fs, commitStatus.getPath(), fs, targetFilePath, false, fs.getConf()); } } else { logger.info("The job has 0 partition to copy."); } // Create the _SUCCESS tag Path successTagPath = new Path(outputDir + "/_SUCCESS"); if (!fs.exists(successTagPath)) { logger.info(String.format("Creating _SUCCESS under targetBasePath: $s", outputDir)); fs.createNewFile(successTagPath); } }
From source file:com.yolodata.tbana.testutils.FileSystemTestUtils.java
License:Open Source License
public static Path createEmptyFile(FileSystem fs, String extension) throws IOException { Path file = new Path(FileTestUtils.getRandomTestFilepath().concat("." + extension)); assertTrue(fs.createNewFile(file)); return fs.getFileStatus(file).getPath(); }
From source file:com.yolodata.tbana.testutils.FileSystemTestUtils.java
License:Open Source License
public static Path createEmptyFile(FileSystem fs, Path location, String extension) throws IOException { Path file = HadoopFileTestUtils.createPath(location.toString(), FileTestUtils.getRandomFilename(extension)); assertTrue(fs.createNewFile(file)); return fs.getFileStatus(file).getPath(); }
From source file:cz.muni.fi.xfabian7.bp.mgrid.HdfsStorageBucket.java
/** * Create a new HDFS file with the given path * * @param path// ww w . j av a 2 s . co m */ public void createFile(Path path) { try { FileSystem fs = getFileSystem(); // Check if the file already exists if (fs.exists(path)) { System.out.println("File " + path + " already exists"); // fs.close(); return; } fs.createNewFile(path); // fs.close(); } catch (IOException ex) { Logger.getLogger(HdfsStorageBucket.class.getName()).log(Level.SEVERE, null, ex); } }
From source file:gobblin.compaction.mapreduce.conditions.RecompactionConditionTest.java
License:Apache License
@Test public void testRecompactionConditionBasedOnFileCount() { try {//from w w w. j a v a 2 s .com Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); fs.delete(outputLatePath, true); fs.mkdirs(outputLatePath); RecompactionConditionFactory factory = new RecompactionConditionBasedOnFileCount.Factory(); RecompactionCondition conditionBasedOnFileCount = factory.createRecompactionCondition(dataset); DatasetHelper helper = new DatasetHelper(dataset, fs, Lists.newArrayList("avro")); fs.createNewFile(new Path(outputLatePath, new Path("1.avro"))); fs.createNewFile(new Path(outputLatePath, new Path("2.avro"))); Assert.assertEquals(conditionBasedOnFileCount.isRecompactionNeeded(helper), false); fs.createNewFile(new Path(outputLatePath, new Path("3.avro"))); Assert.assertEquals(conditionBasedOnFileCount.isRecompactionNeeded(helper), true); fs.delete(outputLatePath, true); } catch (Exception e) { e.printStackTrace(); } }
From source file:gobblin.data.management.trash.Trash.java
License:Apache License
protected void ensureTrashLocationExists(FileSystem fs, Path trashLocation) throws IOException { if (fs.exists(trashLocation)) { if (!fs.isDirectory(trashLocation)) { throw new IOException(String.format("Trash location %s is not a directory.", trashLocation)); }/*from w w w .j a v a 2 s.c o m*/ if (!fs.exists(new Path(trashLocation, TRASH_IDENTIFIER_FILE))) { // If trash identifier file is not present, directory might have been created by user. // Add trash identifier file only if directory is empty. if (fs.listStatus(trashLocation).length > 0) { throw new IOException(String.format( "Trash directory %s exists, but it does not look like a trash directory. " + "File: %s missing and directory is not empty.", trashLocation, TRASH_IDENTIFIER_FILE)); } else if (!fs.createNewFile(new Path(trashLocation, TRASH_IDENTIFIER_FILE))) { throw new IOException(String.format("Failed to create file %s in existing trash directory %s.", TRASH_IDENTIFIER_FILE, trashLocation)); } } } else if (!(fs.mkdirs(trashLocation.getParent(), ALL_PERM) && fs.mkdirs(trashLocation, PERM) && fs.createNewFile(new Path(trashLocation, TRASH_IDENTIFIER_FILE)))) { // Failed to create directory or create trash identifier file. throw new IOException("Failed to create trash directory at " + trashLocation.toString()); } }
From source file:gobblin.data.management.trash.TrashFactoryTest.java
License:Apache License
@Test public void test() throws IOException { FileSystem fs = mock(FileSystem.class); Path homeDirectory = new Path("/home/directory"); Path trashDirectory = new Path(homeDirectory, Trash.DEFAULT_TRASH_DIRECTORY); Path trashIdentifierFile = new Path(trashDirectory, Trash.TRASH_IDENTIFIER_FILE); when(fs.getHomeDirectory()).thenReturn(homeDirectory); when(fs.exists(trashDirectory)).thenReturn(true); when(fs.exists(trashIdentifierFile)).thenReturn(true); when(fs.listStatus(trashDirectory)).thenReturn(new FileStatus[] {}); when(fs.isDirectory(trashDirectory)).thenReturn(true); when(fs.mkdirs(any(Path.class))).thenReturn(true); when(fs.mkdirs(any(Path.class), any(FsPermission.class))).thenReturn(true); when(fs.createNewFile(any(Path.class))).thenReturn(true); when(fs.makeQualified(any(Path.class))).thenAnswer(new Answer<Path>() { @Override/* w ww . j a v a 2s . co m*/ public Path answer(InvocationOnMock invocation) throws Throwable { return (Path) invocation.getArguments()[0]; } }); Properties properties; properties = getBaseProperties(trashDirectory); Assert.assertTrue(TrashFactory.createTrash(fs, properties) instanceof Trash); Assert.assertTrue(TrashFactory.createProxiedTrash(fs, properties) instanceof ProxiedTrash); properties = getBaseProperties(trashDirectory); properties.setProperty(TrashFactory.SIMULATE, Boolean.toString(true)); Assert.assertTrue(TrashFactory.createTrash(fs, properties) instanceof MockTrash); Assert.assertTrue(TrashFactory.createProxiedTrash(fs, properties) instanceof MockTrash); properties = getBaseProperties(trashDirectory); properties.setProperty(TrashFactory.TRASH_TEST, Boolean.toString(true)); Assert.assertTrue(TrashFactory.createTrash(fs, properties) instanceof TestTrash); Assert.assertTrue(TrashFactory.createProxiedTrash(fs, properties) instanceof TestTrash); properties = getBaseProperties(trashDirectory); properties.setProperty(TrashFactory.SKIP_TRASH, Boolean.toString(true)); Assert.assertTrue(TrashFactory.createTrash(fs, properties) instanceof ImmediateDeletionTrash); Assert.assertTrue(TrashFactory.createProxiedTrash(fs, properties) instanceof ImmediateDeletionTrash); }
From source file:gobblin.filesystem.InstrumentedHDFSFileSystemTest.java
License:Open Source License
/** * This test is disabled because it requires a local hdfs cluster at localhost:8020, which requires installation and setup. * Changes to {@link InstrumentedHDFSFileSystem} should be followed by a manual run of this tests. * * TODO: figure out how to fully automate this test. * @throws Exception//from w ww. j a v a2s . c o m */ @Test(enabled = false) public void test() throws Exception { FileSystem fs = FileSystem.get(new URI("instrumented-hdfs://localhost:8020"), new Configuration()); String name = UUID.randomUUID().toString(); fs.mkdirs(new Path("/tmp")); // Test absolute paths Path absolutePath = new Path("/tmp", name); Assert.assertFalse(fs.exists(absolutePath)); fs.createNewFile(absolutePath); Assert.assertTrue(fs.exists(absolutePath)); Assert.assertEquals(fs.getFileStatus(absolutePath).getLen(), 0); fs.delete(absolutePath, false); Assert.assertFalse(fs.exists(absolutePath)); // Test fully qualified paths Path fqPath = new Path("instrumented-hdfs://localhost:8020/tmp", name); Assert.assertFalse(fs.exists(fqPath)); fs.createNewFile(fqPath); Assert.assertTrue(fs.exists(fqPath)); Assert.assertEquals(fs.getFileStatus(fqPath).getLen(), 0); fs.delete(fqPath, false); Assert.assertFalse(fs.exists(fqPath)); }