Example usage for org.apache.hadoop.fs FileSystem close

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem close.

Prototype

@Override
public void close() throws IOException 

Source Link

Document

Close this FileSystem instance.

Usage

From source file:com.netflix.bdp.s3mper.alert.impl.AlertJanitor.java

License:Apache License

/**
 * Writes out logs to the given path as a separate JSON message per line.
 * /*w ww. j  av  a2 s  .com*/
 * @param queue
 * @param path
 * @throws IOException 
 */
public void writeLogs(String queue, Path path) throws IOException {
    FileSystem fs = FileSystem.get(path.toUri(), conf);
    DataOutputStream fout = fs.create(path);

    do {
        List<Message> messages = pull(queue, batchCount);

        if (messages.isEmpty()) {
            break;
        }

        for (Message m : messages) {
            fout.write((m.getBody().replaceAll("[\n|\r]", " ") + "\n").getBytes("UTF8"));
        }

        delete(queue, messages);
    } while (true);

    fout.close();
    fs.close();
}

---

From source file:com.philiphubbard.digraph.MRBuildVerticesTest.java

License:Open Source License

private static void setupTest(Configuration conf) throws IOException {
    FileSystem fileSystem = FileSystem.get(conf);

    Path path = new Path(testInput);
    if (fileSystem.exists(path))
        fileSystem.delete(path, true);/*from w  w w .j  a v a  2s  .com*/

    ArrayList<MRVertex> vertices = new ArrayList<MRVertex>();

    MRVertex v0 = new MRVertex(0, conf);
    v0.addEdgeTo(2);
    vertices.add(v0);

    MRVertex v1 = new MRVertex(1, conf);
    v1.addEdgeTo(2);
    vertices.add(v1);

    MRVertex v2 = new MRVertex(2, conf);
    v2.addEdgeTo(3);
    vertices.add(v2);

    MRVertex v3 = new MRVertex(3, conf);
    v3.addEdgeTo(4);
    vertices.add(v3);

    MRVertex v4 = new MRVertex(4, conf);
    v4.addEdgeTo(5);
    v4.addEdgeTo(6);
    vertices.add(v4);

    MRVertex v5 = new MRVertex(5, conf);
    vertices.add(v5);

    MRVertex v6 = new MRVertex(6, conf);
    v6.addEdgeTo(7);
    vertices.add(v6);

    MRVertex v7 = new MRVertex(7, conf);
    vertices.add(v7);

    FSDataOutputStream out = fileSystem.create(path);
    for (MRVertex vertex : vertices) {
        Text text = vertex.toText(MRVertex.EdgeFormat.EDGES_TO);
        byte[] bytes = text.copyBytes();
        for (byte b : bytes)
            out.write(b);
        out.write('\n');
    }
    out.close();

    fileSystem.close();
}

---

From source file:com.philiphubbard.digraph.MRBuildVerticesTest.java

License:Open Source License

private static void cleanupTest(Configuration conf) throws IOException {
    FileSystem fileSystem = FileSystem.get(conf);

    ArrayList<MRVertex> branch = new ArrayList<MRVertex>();

    FileStatus[] branchFiles = fileSystem.listStatus(new Path(testOutput + "/branch"));
    for (FileStatus status : branchFiles)
        readVertices(status, branch, conf);

    for (MRVertex vertex : branch)
        System.out.println(vertex.toDisplayString());

    ArrayList<MRVertex> chain = new ArrayList<MRVertex>();

    FileStatus[] chainFiles = fileSystem.listStatus(new Path(testOutput + "/chain"));
    for (FileStatus status : chainFiles)
        readVertices(status, chain, conf);

    for (MRVertex vertex : chain)
        System.out.println(vertex.toDisplayString());

    fileSystem.delete(new Path(testInput), true);
    fileSystem.delete(new Path(testOutput), true);

    fileSystem.close();
}

---

From source file:com.philiphubbard.digraph.MRCompressChainsTest.java

License:Open Source License

private static void setupTest(Configuration conf) throws IOException {
    FileSystem fileSystem = FileSystem.get(conf);

    Path path = new Path(testInput);
    if (fileSystem.exists(path))
        fileSystem.delete(path, true);/* ww  w.  j  av a2s  .c o  m*/

    ArrayList<MRVertex> vertices = new ArrayList<MRVertex>();
    for (int i = 0; i < 60; i++) {
        MRVertex vertex = new MRVertex(i, conf);
        vertices.add(vertex);
        if (i % 20 != 19)
            vertex.addEdgeTo(i + 1);
    }

    SequenceFile.Writer writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(path),
            SequenceFile.Writer.keyClass(IntWritable.class),
            SequenceFile.Writer.valueClass(BytesWritable.class));
    for (MRVertex vertex : vertices)
        writer.append(new IntWritable(vertex.getId()), vertex.toWritable(MRVertex.EdgeFormat.EDGES_TO));
    writer.close();

    fileSystem.close();
}

---

From source file:com.philiphubbard.digraph.MRCompressChainsTest.java

License:Open Source License

private static void cleanupTest(Configuration conf) throws IOException {
    FileSystem fileSystem = FileSystem.get(conf);

    ArrayList<MRVertex> vertices = new ArrayList<MRVertex>();

    FileStatus[] files = fileSystem.listStatus(new Path(testOutput));
    for (FileStatus status : files) {
        Path path = status.getPath();
        if (path.getName().startsWith("part")) {
            System.out.println(path);

            SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(path));
            IntWritable key = new IntWritable();
            BytesWritable value = new BytesWritable();
            while (reader.next(key, value))
                vertices.add(new MRVertex(value, conf));
            reader.close();/*w  ww.  j  a  v a  2  s. c  om*/
        }
    }

    for (MRVertex vertex : vertices)
        System.out.println(vertex.toDisplayString());

    fileSystem.delete(new Path(testInput), true);
    fileSystem.delete(new Path(testOutput), true);

    fileSystem.close();
}

---

From source file:com.philiphubbard.sabe.MRAssembler.java

License:Open Source License

public boolean run(Path inputPath, Path outputPath)
        throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();

    // Job.getInstance() copies the Configuration argument, so set its properties first.

    conf.setBoolean(MRVertex.CONFIG_ALLOW_EDGE_MULTIPLES, true);
    conf.setBoolean(MRVertex.CONFIG_COMPRESS_CHAIN_MULTIPLES_MUST_MATCH, false);
    conf.setInt(MRMerVertex.CONFIG_MER_LENGTH, vertexMerLength);
    conf.setBoolean(MRBuildVertices.CONFIG_PARTITION_BRANCHES_CHAINS, true);
    conf.setInt(MRBuildVertices.CONFIG_COVERAGE, coverage);
    conf.setInt(MRCompressChains.CONFIG_TERMINATION_COUNT, 1);

    Job buildJob = Job.getInstance(conf);
    buildJob.setJobName("mrassemblerbuild");

    Path buildInputPath = inputPath;
    Path buildOutputPath = new Path("sabe.MRAssemblerTmp");

    System.out.println("sabe.MRAssembler starting vertex construction");

    MRBuildMerVertices.setupJob(buildJob, buildInputPath, buildOutputPath);

    if (!buildJob.waitForCompletion(true))
        return false;

    ////from   w  w w . ja va 2s. c o  m

    Path compressInputPath = new Path(buildOutputPath.toString() + "/chain");
    Path compressOutputPath = new Path(buildOutputPath.toString() + "/chainCompress");

    int iter = 0;
    boolean keepGoing = true;
    MRCompressChains.beginIteration();
    while (keepGoing) {
        Job compressJob = Job.getInstance(conf);
        compressJob.setJobName("mrassemblercompress");

        System.out.println("sabe.MRAssembler starting compression iteration " + iter);

        MRCompressMerChains.setupIterationJob(compressJob, compressInputPath, compressOutputPath);

        if (!compressJob.waitForCompletion(true))
            System.exit(1);

        iter++;
        keepGoing = MRCompressChains.continueIteration(compressJob, compressInputPath, compressOutputPath);
    }

    System.out.println("sabe.MRAssembler made " + iter + " compression iterations");

    //

    Path branchPath = new Path(buildOutputPath.toString() + "/branch");
    Path chainPath = compressOutputPath;

    FileSystem fileSystem = FileSystem.get(conf);

    Graph graph = buildCompressedGraph(conf, fileSystem, branchPath, chainPath);
    if (graph != null) {
        ArrayList<String> result = graph.assemble();

        FSDataOutputStream out = fileSystem.create(outputPath);
        for (String seq : result) {
            out.writeBytes(seq);
            out.writeBytes("\n");
        }
    }

    //

    fileSystem.delete(buildOutputPath, true);

    fileSystem.close();

    return true;
}

---

From source file:com.philiphubbard.sabe.MRAssemblerTest1.java

License:Open Source License

private static void setupTest(Configuration conf) throws IOException {
    FileSystem fileSystem = FileSystem.get(conf);

    Path path = new Path(testInput);
    if (fileSystem.exists(path))
        fileSystem.delete(path, true);//from   w ww. j  a  v  a 2s .  c  o m

    ArrayList<Text> reads = new ArrayList<Text>();

    // Goal: AATTCGGCCTTCGGCAT

    reads.add(new Text("AATTCGGC\n"));
    reads.add(new Text("CTTCGGCAT\n"));

    reads.add(new Text("AATT\n"));
    reads.add(new Text("CGGCCTTCGGCAT\n"));

    reads.add(new Text("AATTCGGCCTTCG\n"));
    reads.add(new Text("GCAT\n"));

    FSDataOutputStream out = fileSystem.create(path);
    for (Text read : reads) {
        byte[] bytes = read.copyBytes();
        for (byte b : bytes)
            out.write(b);
    }
    out.close();

    fileSystem.close();
}

---

From source file:com.philiphubbard.sabe.MRAssemblerTest1.java

License:Open Source License

private static void cleanupTest(Configuration conf) throws IOException {
    FileSystem fileSystem = FileSystem.get(conf);

    fileSystem.delete(new Path(testInput), true);
    fileSystem.delete(new Path(testOutput), true);

    fileSystem.close();
}

---

From source file:com.philiphubbard.sabe.MRAssemblerTest2.java

License:Open Source License

private static void setupTest(Configuration conf) throws IOException {
    FileSystem fileSystem = FileSystem.get(conf);

    Path path = new Path(testInput);
    if (fileSystem.exists(path))
        fileSystem.delete(path, true);/* w  ww.j  a v  a 2 s . co m*/

    ArrayList<Text> reads = new ArrayList<Text>();

    // The expected result:
    // CCCTTTCTGTTGACCCATCATTGTTTAGTAACCCGCGGGATGCCTGGCAGACCCGCGGGACGATCTCCTCTGACCCATCATCGAAATTCC
    // Note that it has the following pattern:
    // segment 0: CCCTTTCTGT 
    // segment 1, which will be repeated: TGACCCATCA 
    // segment 2: TTGTTTAGTA 
    // segment 3, which will be repeated: ACCCGCGGGA 
    // segment 4: TGCCTGGCAG 
    // segment 3, again: ACCCGCGGGA 
    // segment 5: CGATCTCCTC
    // segment 1, again: TGACCCATCA 
    // segment 6: TCGAAATTCC

    reads.add(new Text("CCCTTTC\n"));
    // Error: initial T omitted.
    reads.add(new Text("GTTGACCCATCATTGTTTAGTAACCCGCGGGATGCCTGGCAGACC"));
    reads.add(new Text("CGCGGGACGAT\n"));
    // Error: final C omitted.
    reads.add(new Text("CTCCTCTGACCCATCATCGAAATTC\n"));

    reads.add(new Text("CCCTTTCTGTTGACCCAT\n"));
    // Error: final C replaced with G.
    reads.add(new Text("CATTGTTTAGTAACCCGCGGGATGCCTGGCAGACG\n"));
    reads.add(new Text("CGCGGGACGATCTCCTCTGACCCATCATCGAAATTCC\n"));

    // Error: C at index 14 replaced with A.
    reads.add(new Text("CCCTTTCTGTTGACACATCATTGTTTAGTAAC"));
    reads.add(new Text("CCGCGGGATGCC\n"));
    // Error: C at index 25 omitted.
    reads.add(new Text("TGGCAGACCCGCGGGACGATCTCCTTGACCCATCATCGAAATTCC\n"));

    reads.add(new Text("CCCTTTCTGTTGACCCATCATTGTTTAGTAACCCGCGGGATGCCTG\n"));
    // Error: G at index 10 replaced with T.
    reads.add(new Text("GCAGACCCGCTGGACGA\n"));
    reads.add(new Text("TCTCCTCTGACCCATCATCGAAATTCC\n"));

    reads.add(new Text("CCCTTTCTGTTGACCCATCATTGTTTAGTAACCCGCGGGATGC"));
    // Error: final G omitted.
    reads.add(new Text("CTGGCAGACCCGC\n"));
    reads.add(new Text("GGACGATCTCCTCT\n"));
    // Error: CG at index 10 transposed to GC
    reads.add(new Text("GACCCATCATCGAAATTCC\n"));

    FSDataOutputStream out = fileSystem.create(path);
    for (Text read : reads) {
        byte[] bytes = read.copyBytes();
        for (byte b : bytes)
            out.write(b);
    }
    out.close();

    fileSystem.close();
}

---

From source file:com.pivotal.hawq.mapreduce.MapReduceClusterDriver.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    if (args.length != 3 && args.length != 4) {
        System.err.printf("Usage: %s [generic options] <tableName> <dburl> <output> [<mapper_classname>]\n",
                getClass().getSimpleName());
        ToolRunner.printGenericCommandUsage(System.err);
        return -1;
    }/* ww  w.  j a v  a  2 s . com*/

    String tableName = args[0];
    String dbUrl = args[1];
    Path outputPath = new Path(args[2]);
    Class<? extends Mapper> mapperClass = (args.length == 3) ? HAWQTableMapper.class
            : (Class<? extends Mapper>) Class.forName(args[3]);

    // delete previous output
    FileSystem fs = FileSystem.get(getConf());
    if (fs.exists(outputPath))
        fs.delete(outputPath, true);
    fs.close();

    Job job = new Job(getConf(), "job_read_" + tableName);
    job.setJarByClass(MapReduceClusterDriver.class);

    job.setInputFormatClass(HAWQInputFormat.class);
    HAWQInputFormat.setInput(job.getConfiguration(), dbUrl, null, null, tableName);
    FileOutputFormat.setOutputPath(job, outputPath);

    job.setMapperClass(mapperClass);
    job.setReducerClass(HAWQTableReducer.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    return job.waitForCompletion(true) ? 0 : 1;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• »