Example usage for org.apache.hadoop.fs FileSystem createNewFile

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem createNewFile.

Prototype

public boolean createNewFile(Path f) throws IOException 

Source Link

Document

Creates the given Path as a brand-new zero-length file.

Usage

From source file:org.apache.nutch.admin.scheduling.AdminCrawl.java

License:Apache License

private void fetchSegment(Path segment, Configuration configuration) throws IOException {
    FileSystem fileSystem = FileSystem.get(configuration);
    Path running = new Path(segment, "fetch.running");
    fileSystem.createNewFile(running);
    Fetcher fetcher = new Fetcher(configuration);
    fetcher.fetch(segment, configuration.getInt("fetcher.threads.fetch", 10), true);
    fileSystem.createNewFile(new Path(segment, "fetch.done"));
    if (configuration.getBoolean("fetcher.parse", true)) {
        fileSystem.createNewFile(new Path(segment, "parse.done"));
    }/*w w w  .j ava2s  .co  m*/

}

---

From source file:org.apache.nutch.admin.scheduling.AdminCrawl.java

License:Apache License

private Path generateSegment(Path crawldbFile, Path segments, Configuration configuration) throws IOException {
    FileSystem system = FileSystem.get(configuration);
    Path runningGenerateSegment = new Path(segments, "generate.running");
    Path runningGenerateDB = new Path(crawldbFile, "generate.running");
    system.createNewFile(runningGenerateSegment);
    system.createNewFile(runningGenerateDB);
    Generator generator = new Generator(configuration);
    // Path segment = generator.generate(crawldbFile, segments);
    long topN = Long.MAX_VALUE;
    Path segment = generator.generate(crawldbFile, segments, -1, topN, System.currentTimeMillis());
    return segment;
}

---

From source file:org.apache.oozie.action.hadoop.TestFsActionExecutor.java

License:Apache License

public void testDeleteWithGlob() throws Exception {

    FsActionExecutor ae = new FsActionExecutor();
    FileSystem fs = getFileSystem();
    Context context = createContext("<fs/>");
    Path basePath = new Path(getFsTestCaseDir(), "2010");
    fs.mkdirs(basePath);/*  w w w .j  a  v  a2 s.c om*/
    fs.mkdirs(new Path(basePath, "10"));
    fs.createNewFile(new Path(basePath + "/10/newfile1"));
    fs.createNewFile(new Path(basePath + "/10/newfile2"));
    fs.mkdirs(new Path(basePath, "11"));
    fs.createNewFile(new Path(basePath + "/11/newfile3"));
    fs.mkdirs(new Path(basePath, "12"));
    fs.createNewFile(new Path(basePath + "/12/newfile4"));

    Path globPath = new Path(basePath + "/1{0,1}/*");
    ae.delete(context, globPath);
    assertFalse(fs.exists(new Path(basePath + "/10/newfile1")));
    assertFalse(fs.exists(new Path(basePath + "/10/newfile2")));
    assertFalse(fs.exists(new Path(basePath + "/11/newfile3")));
    assertTrue(fs.exists(new Path(basePath + "/12/newfile4")));

    fs.delete(basePath, true);
}

---

From source file:org.apache.oozie.action.hadoop.TestFsActionExecutor.java

License:Apache License

public void testMove() throws Exception {
    FsActionExecutor ae = new FsActionExecutor();
    FileSystem fs = getFileSystem();

    Path source = new Path(getFsTestCaseDir(), "source");
    Path target = new Path(getFsTestCaseDir(), "target");
    Context context = createContext("<fs/>");

    fs.mkdirs(source);/*from ww  w . j  av a  2  s.  co m*/
    fs.createNewFile(new Path(source + "/newfile1"));
    fs.mkdirs(target);

    String dest = target.toUri().getPath();
    Path destPath = new Path(dest);
    ae.move(context, new Path(source + "/newfile1"), destPath, false);

    assertTrue(!fs.exists(new Path(source + "/newfile1")));
    assertTrue(fs.exists(target));

    try {
        ae.move(context, new Path(source + "/newfile1"), destPath, false);
        fail();
    } catch (ActionExecutorException ex) {
        assertEquals("FS006", ex.getErrorCode());
    }

    fs.mkdirs(source);
    fs.createNewFile(new Path(source + "/newfile"));
    Path complexTarget = new Path(target + "/a/b");
    fs.mkdirs(complexTarget);

    ae.move(context, source, complexTarget, false);
    assertTrue(fs.exists(new Path(complexTarget + "/" + source.getName())));

    fs.mkdirs(source);
    try {
        ae.move(context, source, new Path(target.toUri().getScheme() + "://foo/" + destPath), false);
        fail();
    } catch (ActionExecutorException ex) {
        assertEquals("FS007", ex.getErrorCode());
    }
    fs.delete(source, true);
    ae.move(context, source, new Path(target.toUri().getPath()), true);

    fs.mkdirs(source);
    fs.delete(target, true);
    ae.move(context, source, new Path(target.toUri().getPath()), true);
    assertTrue(!fs.exists(source));
    assertTrue(fs.exists(target));
}

---

From source file:org.apache.oozie.action.hadoop.TestFsActionExecutor.java

License:Apache License

public void testMoveWithGlob() throws Exception {
    FsActionExecutor ae = new FsActionExecutor();
    FileSystem fs = getFileSystem();
    Path source = new Path(getFsTestCaseDir(), "source");
    Path target = new Path(getFsTestCaseDir(), "target");
    Context context = createContext("<fs/>");

    // Test simple example of glob
    fs.mkdirs(source);//from  www  .  j ava2 s  .co  m
    fs.mkdirs(target);
    fs.createNewFile(new Path(source, "newfile1"));
    fs.createNewFile(new Path(source, "newfile2"));

    ae.move(context, new Path(source.toString() + "/*"), target, false);
    assertTrue(fs.exists(new Path(target, "newfile1")));
    assertTrue(fs.exists(new Path(target, "newfile2")));

    // Test another example of glob
    fs.delete(target, true);
    fs.mkdirs(target);
    fs.mkdirs(new Path(source + "/2010"));
    fs.mkdirs(new Path(source + "/2011"));
    fs.mkdirs(new Path(source + "/2012"));
    fs.mkdirs(new Path(source + "/2010/10"));
    fs.mkdirs(new Path(source + "/2010/11"));
    fs.createNewFile(new Path(source + "/2010/10/newfile1"));
    fs.createNewFile(new Path(source + "/2010/11/newfile2"));
    fs.mkdirs(new Path(source + "/2011/09"));
    fs.mkdirs(new Path(source + "/2011/10"));
    fs.createNewFile(new Path(source + "/2011/09/newfile3"));
    fs.createNewFile(new Path(source + "/2011/10/newfile4"));

    ae.move(context, new Path(source.toString() + "/201[0-1]/1{0,1}/*"), target, false);
    assertTrue(fs.exists(new Path(target.toString() + "/newfile1")));
    assertTrue(fs.exists(new Path(target.toString() + "/newfile2")));
    assertFalse(fs.exists(new Path(target.toString() + "/newfile3")));
    assertTrue(fs.exists(new Path(target.toString() + "/newfile4")));

    fs.delete(new Path(source + "/2010"), true);
    fs.delete(new Path(source + "/2011"), true);
    fs.delete(new Path(source + "/2012"), true);

    // Catch exception when trying to move multiple files (match glob) to
    // the same name which doesn't exist
    fs.delete(target, true);
    try {
        ae.move(context, new Path(source.toString() + "/*"), target, true);
    } catch (ActionExecutorException ex) {
        assertEquals("FS012", ex.getErrorCode());
    }

    // Catch exception when trying to move multiple files (match glob) to
    // the same file name which exists
    fs.delete(target, true);
    Path targetFile = new Path(target, "newfile1");
    fs.createNewFile(targetFile);
    try {
        ae.move(context, new Path(source.toString() + "/*"), targetFile, true);
    } catch (ActionExecutorException ex) {
        assertEquals("FS012", ex.getErrorCode());
    }
}

---

From source file:org.apache.oozie.action.hadoop.TestFsActionExecutor.java

License:Apache License

private void createTestDirForChmod(Path basePath, FileSystem fs) throws Exception {
    fs.mkdirs(basePath);/*ww w .jav a 2  s.c o m*/
    fs.mkdirs(new Path(basePath, "10"));
    fs.mkdirs(new Path(basePath + "/10/dir1"));
    fs.createNewFile(new Path(basePath + "/10/dir1/file1"));
    fs.mkdirs(new Path(basePath + "/10/dir2"));
    fs.mkdirs(new Path(basePath, "11"));
    fs.mkdirs(new Path(basePath + "/11/dir3"));
    fs.mkdirs(new Path(basePath, "12"));
    fs.setPermission(new Path(basePath, "10"), FsPermission.valueOf("-rwxrwxrwx"));
    fs.setPermission(new Path(basePath + "/10/dir1"), FsPermission.valueOf("-rwxrwxrwx"));
    fs.setPermission(new Path(basePath + "/10/dir2"), FsPermission.valueOf("-rwxrwxrwx"));
    fs.setPermission(new Path(basePath + "/10/dir1/file1"), FsPermission.valueOf("-rw-rw-rw-"));
    fs.setPermission(new Path(basePath, "11"), FsPermission.valueOf("-rwxrwxrwx"));
    fs.setPermission(new Path(basePath + "/11/dir3"), FsPermission.valueOf("-rwxrwxrwx"));
    fs.setPermission(new Path(basePath, "12"), FsPermission.valueOf("-rwxrwxrwx"));
}

---

From source file:org.apache.oozie.action.hadoop.TestFsActionExecutor.java

License:Apache License

public void testTouchz() throws Exception {
    FsActionExecutor ae = new FsActionExecutor();
    FileSystem fs = getFileSystem();

    Path dir = new Path(getFsTestCaseDir(), "dir1");
    Path f1 = new Path(dir + "/newfile1");
    Path f2 = new Path(dir + "/newfile2");
    Context context = createContext("<fs/>");

    fs.mkdirs(dir);/*from w w w  .  j  a v a 2 s . c om*/
    fs.createNewFile(f1);

    assertTrue(fs.exists(f1));
    assertTrue(fs.getFileStatus(f1).getLen() == 0);

    ae.touchz(context, f1);
    ae.touchz(context, f2);

    assertTrue(fs.exists(f1));
    assertTrue(fs.exists(f2));

    FileStatus fs1 = fs.getFileStatus(f1);
    FileStatus fs2 = fs.getFileStatus(f2);

    assertFalse(fs1.isDir());
    assertFalse(fs2.isDir());
    assertTrue(fs1.getLen() == 0);
    assertTrue(fs2.getLen() == 0);

    try {
        ae.touchz(context, dir);
        fail();
    } catch (Exception ex) {
        assertTrue(ex.getMessage().contains("is a directory"));
    }

    //Test touchz on a non-zero length file
    Path f3 = new Path(dir + "/newfile3");
    Writer writer = new OutputStreamWriter(fs.create(f3));
    writer.write("This is not a zero length file");
    writer.close();

    try {
        ae.touchz(context, f3);
        fail();
    } catch (Exception ex) {
        assertTrue(ex.getMessage().contains("must be a zero-length file"));
    }
}

---

From source file:org.apache.oozie.action.hadoop.TestFsActionExecutor.java

License:Apache License

public void testDoOperations() throws Exception {
    FsActionExecutor ae = new FsActionExecutor();
    FileSystem fs = getFileSystem();

    Path mkdir = new Path(getFsTestCaseDir(), "mkdir");
    Path delete = new Path(getFsTestCaseDir(), "delete");
    fs.mkdirs(delete);//from  w w  w.  j a  v  a  2  s .co m
    Path source = new Path(getFsTestCaseDir(), "source");
    fs.mkdirs(source);
    Path target = new Path(new Path(getFsTestCaseDir(), "target").toUri().getPath());
    Path chmod1 = new Path(getFsTestCaseDir(), "chmod1");
    fs.mkdirs(chmod1);
    Path child1 = new Path(chmod1, "child1");
    fs.mkdirs(child1);
    Path chmod2 = new Path(getFsTestCaseDir(), "chmod2");
    fs.mkdirs(chmod2);
    Path child2 = new Path(chmod2, "child2");
    fs.mkdirs(child2);
    Path newFile1 = new Path(mkdir + "newFile1");
    Path newFile2 = new Path(mkdir + "newFile2");
    fs.createNewFile(newFile1);
    Path chmod3 = new Path(getFsTestCaseDir(), "chmod3");
    fs.mkdirs(chmod3);
    Path child3 = new Path(chmod3, "child3");
    fs.mkdirs(child3);
    Path grandchild3 = new Path(child3, "grandchild1");
    fs.mkdirs(grandchild3);

    String str = MessageFormat.format(
            "<root><mkdir path=''{0}''/>" + "<delete path=''{1}''/>" + "<move source=''{2}'' target=''{3}''/>"
                    + "<chmod path=''{4}'' permissions=''-rwxrwxrwx''/>"
                    + "<chmod path=''{5}'' permissions=''-rwxrwx---'' dir-files=''false''/>"
                    + "<touchz path=''{6}''/>" + "<touchz path=''{7}''/>"
                    + "<chmod path=''{8}'' permissions=''-rwxrwx---''> <recursive/> </chmod>" + "</root>",
            mkdir, delete, source, target, chmod1, chmod2, newFile1, newFile2, chmod3);

    Element xml = XmlUtils.parseXml(str);

    ae.doOperations(createContext("<fs/>"), xml);

    assertTrue(fs.exists(mkdir));
    assertFalse(fs.exists(delete));
    assertFalse(fs.exists(source));
    assertTrue(fs.exists(target));
    assertTrue(fs.exists(newFile1));
    assertTrue(fs.exists(newFile2));

    assertEquals("rwxrwxrwx", fs.getFileStatus(chmod1).getPermission().toString());
    assertNotSame("rwxrwxrwx", fs.getFileStatus(child1).getPermission().toString());
    assertEquals("rwxrwx---", fs.getFileStatus(chmod2).getPermission().toString());
    assertNotSame("rwxrwx---", fs.getFileStatus(child2).getPermission().toString());

    assertEquals("rwxrwx---", fs.getFileStatus(child3).getPermission().toString());
    assertEquals("rwxrwx---", fs.getFileStatus(grandchild3).getPermission().toString());

    assertEquals("rwxrwx---", fs.getFileStatus(child3).getPermission().toString());
    assertEquals("rwxrwx---", fs.getFileStatus(grandchild3).getPermission().toString());
}

---

From source file:org.apache.oozie.action.hadoop.TestFsActionExecutor.java

License:Apache License

public void testDoOperationsWithNameNodeElement() throws Exception {
    FsActionExecutor ae = new FsActionExecutor();
    FileSystem fs = getFileSystem();

    Path mkdir = new Path(getFsTestCaseDir(), "mkdir");
    Path mkdirX = new Path(mkdir.toUri().getPath());
    Path delete = new Path(getFsTestCaseDir(), "delete");
    Path deleteX = new Path(delete.toUri().getPath());
    fs.mkdirs(delete);//from  www .java2s  . co  m
    Path source = new Path(getFsTestCaseDir(), "source");
    Path sourceX = new Path(source.toUri().getPath());
    fs.mkdirs(source);
    Path target = new Path(new Path(getFsTestCaseDir(), "target").toUri().getPath());
    Path chmod1 = new Path(getFsTestCaseDir(), "chmod1");
    Path chmod1X = new Path(chmod1.toUri().getPath());
    fs.mkdirs(chmod1);
    Path child1 = new Path(chmod1, "child1");
    fs.mkdirs(child1);
    Path chmod2 = new Path(getFsTestCaseDir(), "chmod2");
    Path chmod2X = new Path(chmod2.toUri().getPath());
    fs.mkdirs(chmod2);
    Path child2 = new Path(chmod2, "child2");
    fs.mkdirs(child2);
    Path newFile1 = new Path(mkdir + "newFile1");
    Path newFile1X = new Path(newFile1.toUri().getPath());
    Path newFile2 = new Path(mkdir + "newFile2");
    Path newFile2X = new Path(newFile2.toUri().getPath());
    fs.createNewFile(newFile1);
    Path chmod3 = new Path(getFsTestCaseDir(), "chmod3");
    Path chmod3X = new Path(chmod3.toUri().getPath());
    fs.mkdirs(chmod3);
    Path child3 = new Path(chmod3, "child3");
    fs.mkdirs(child3);
    Path grandchild3 = new Path(child3, "grandchild1");
    fs.mkdirs(grandchild3);

    String str = MessageFormat.format(
            "<root><name-node>{0}</name-node>" + "<mkdir path=''{1}''/>" + "<delete path=''{2}''/>"
                    + "<move source=''{3}'' target=''{4}''/>"
                    + "<chmod path=''{5}'' permissions=''-rwxrwxrwx''/>"
                    + "<chmod path=''{6}'' permissions=''-rwxrwx---'' dir-files=''false''/>"
                    + "<touchz path=''{7}''/>" + "<touchz path=''{8}''/>"
                    + "<chmod path=''{9}'' permissions=''-rwxrwx---''> <recursive/> </chmod>" + "</root>",
            getNameNodeUri(), mkdirX, deleteX, sourceX, target, chmod1X, chmod2X, newFile1X, newFile2X,
            chmod3X);

    Element xml = XmlUtils.parseXml(str);

    ae.doOperations(createContext("<fs/>"), xml);

    assertTrue(fs.exists(mkdir));
    assertFalse(fs.exists(delete));
    assertFalse(fs.exists(source));
    assertTrue(fs.exists(target));
    assertTrue(fs.exists(newFile1));
    assertTrue(fs.exists(newFile2));

    assertEquals("rwxrwxrwx", fs.getFileStatus(chmod1).getPermission().toString());
    assertNotSame("rwxrwxrwx", fs.getFileStatus(child1).getPermission().toString());
    assertEquals("rwxrwx---", fs.getFileStatus(chmod2).getPermission().toString());
    assertNotSame("rwxrwx---", fs.getFileStatus(child2).getPermission().toString());

    assertEquals("rwxrwx---", fs.getFileStatus(child3).getPermission().toString());
    assertEquals("rwxrwx---", fs.getFileStatus(grandchild3).getPermission().toString());

    assertEquals("rwxrwx---", fs.getFileStatus(child3).getPermission().toString());
    assertEquals("rwxrwx---", fs.getFileStatus(grandchild3).getPermission().toString());
}

---

From source file:org.apache.oozie.action.hadoop.TestFsActionExecutor.java

License:Apache License

public void testSubmit() throws Exception {
    FsActionExecutor ae = new FsActionExecutor();
    FileSystem fs = getFileSystem();

    Path mkdir = new Path(getFsTestCaseDir(), "mkdir");
    Path delete = new Path(getFsTestCaseDir(), "delete");
    fs.mkdirs(delete);//from  www  .  j a va2  s  . c  o  m
    Path source = new Path(getFsTestCaseDir(), "source");
    fs.mkdirs(source);
    Path target = new Path(new Path(getFsTestCaseDir(), "target").toUri().getPath());
    Path chmod1 = new Path(getFsTestCaseDir(), "chmod1");
    fs.mkdirs(chmod1);
    Path child1 = new Path(chmod1, "child1");
    fs.mkdirs(child1);
    Path chmod2 = new Path(getFsTestCaseDir(), "chmod2");
    fs.mkdirs(chmod2);
    Path child2 = new Path(chmod2, "child2");
    fs.mkdirs(child2);
    Path newFile1 = new Path(mkdir + "newFile1");
    Path newFile2 = new Path(mkdir + "newFile2");
    fs.createNewFile(newFile1);

    String actionXml = MessageFormat.format(
            "<fs><mkdir path=''{0}''/>" + "<delete path=''{1}''/>" + "<move source=''{2}'' target=''{3}''/>"
                    + "<chmod path=''{4}'' permissions=''-rwxrwxrwx''/>"
                    + "<chmod path=''{5}'' permissions=''-rwxrwx---'' dir-files=''false''/>"
                    + "<touchz path=''{6}''/>" + "<touchz path=''{7}''/>" + "</fs>",
            mkdir, delete, source, target, chmod1, chmod2, newFile1, newFile2);

    Context context = createContext(actionXml);
    WorkflowAction action = context.getAction();

    assertFalse(fs.exists(ae.getRecoveryPath(context)));

    ae.start(context, action);

    assertTrue(fs.exists(ae.getRecoveryPath(context)));

    ae.check(context, context.getAction());
    assertEquals("OK", context.getAction().getExternalStatus());
    assertNull(context.getAction().getData());
    ae.end(context, context.getAction());
    assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());

    assertFalse(fs.exists(ae.getRecoveryPath(context)));

    assertTrue(fs.exists(mkdir));
    assertFalse(fs.exists(delete));
    assertFalse(fs.exists(source));
    assertTrue(fs.exists(target));
    assertTrue(fs.exists(newFile1));
    assertTrue(fs.exists(newFile2));

    assertEquals("rwxrwxrwx", fs.getFileStatus(chmod1).getPermission().toString());
    assertNotSame("rwxrwxrwx", fs.getFileStatus(child1).getPermission().toString());
    assertEquals("rwxrwx---", fs.getFileStatus(chmod2).getPermission().toString());
    assertNotSame("rwxrwx---", fs.getFileStatus(child2).getPermission().toString());

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• »