Example usage for org.apache.hadoop.fs FileSystem get

List of usage examples for org.apache.hadoop.fs FileSystem get

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. get

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem get.

Prototype

public static FileSystem get(Configuration conf) throws IOException

Source Link

Document

Returns the configured FileSystem implementation.

Usage

From source file:boa.functions.BoaAstIntrinsics.java

License:Apache License

private static void openCommentMap() {
    final Configuration conf = new Configuration();
    try {//from ww  w . j av a  2  s  .  co m
        final FileSystem fs = FileSystem.get(conf);
        final Path p = new Path("hdfs://boa-njt/",
                new Path(
                        context.getConfiguration().get("boa.comments.dir",
                                context.getConfiguration().get("boa.input.dir", "repcache/live")),
                        new Path("comments")));
        commentsMap = new MapFile.Reader(fs, p.toString(), conf);
    } catch (final Exception e) {
        e.printStackTrace();
    }
}

From source file:boa.functions.BoaAstIntrinsics.java

License:Apache License

private static void openIssuesMap() {
    final Configuration conf = new Configuration();
    try {//  w w w.  j  a  v  a2 s  . c  o  m
        final FileSystem fs = FileSystem.get(conf);
        final Path p = new Path("hdfs://boa-njt/",
                new Path(
                        context.getConfiguration().get("boa.issues.dir",
                                context.getConfiguration().get("boa.input.dir", "repcache/live")),
                        new Path("issues")));
        issuesMap = new MapFile.Reader(fs, p.toString(), conf);
    } catch (final Exception e) {
        e.printStackTrace();
    }
}

From source file:boa.functions.BoaIntrinsics.java

License:Apache License

/**
 * Given the model URL, deserialize the model and return Model type
 *
 * @param Take URL for the model//from w  w w  . j ava 2  s  .  c o m
 * @return Model type after deserializing
 */
// TODO Take complete URL and then deserialize the model
// FIXME Returning Object as a type, this needs to be changed once we defined Model Type
@FunctionSpec(name = "load", returnType = "Model", formalParameters = { "string" })
public static Object load(final String URL) throws Exception {
    Object unserializedObject = null;
    FSDataInputStream in = null;
    try {
        final Configuration conf = new Configuration();
        final FileSystem fileSystem = FileSystem.get(conf);
        final Path path = new Path("hdfs://boa-njt" + URL);

        if (in != null)
            try {
                in.close();
            } catch (final Exception e) {
                e.printStackTrace();
            }

        in = fileSystem.open(path);
        int numBytes = 0;
        final byte[] b = new byte[(int) fileSystem.getLength(path) + 1];
        long length = 0;

        in.read(b);

        ByteArrayInputStream bin = new ByteArrayInputStream(b);
        ObjectInputStream dataIn = new ObjectInputStream(bin);
        unserializedObject = dataIn.readObject();
        dataIn.close();
    } catch (Exception ex) {
    }
    return unserializedObject;
}

From source file:boostingPL.boosting.BoostingPLFactory.java

License:Open Source License

public static Classifier createBoostingPL(String name, Configuration conf, Path path) throws IOException {
    FileSystem hdfs = FileSystem.get(conf);
    @SuppressWarnings("deprecation")
    SequenceFile.Reader in = new SequenceFile.Reader(hdfs, path, conf);

    IntWritable key = new IntWritable();
    ArrayList<ArrayList<ClassifierWritable>> classifiersW = new ArrayList<ArrayList<ClassifierWritable>>();
    ArrayList<ClassifierWritable> ws = null;
    while (in.next(key)) {
        // key is in order
        if (key.get() + 1 > classifiersW.size()) {
            ws = new ArrayList<ClassifierWritable>();
            classifiersW.add(ws);//  w  w w  .  jav a 2 s  .c o m
        }
        ClassifierWritable value = new ClassifierWritable();
        in.getCurrentValue(value);
        ws.add(value);
    }
    in.close();

    System.out.println("Number of Worker:" + classifiersW.size());
    System.out.println("Number of Iteration:" + classifiersW.get(0).size());
    System.out.println();

    double[][] corWeights = new double[classifiersW.size()][classifiersW.get(0).size()];
    Classifier[][] classifiers = new Classifier[classifiersW.size()][classifiersW.get(0).size()];

    for (int i = 0; i < classifiersW.size(); i++) {
        for (int j = 0; j < classifiersW.get(i).size(); j++) {
            ClassifierWritable c = classifiersW.get(i).get(j);
            classifiers[i][j] = c.getClassifier();
            corWeights[i][j] += c.getCorWeight();
        }
    }

    return createBoostingPL(name, classifiers, corWeights);
}

From source file:boostingPL.MR.AdaBoostPLMapper.java

License:Open Source License

/** create instances header */
protected void setup(Context context) throws IOException, InterruptedException {
    String pathSrc = context.getConfiguration().get("BoostingPL.metadata");
    FileSystem hdfs = FileSystem.get(context.getConfiguration());
    FSDataInputStream dis = new FSDataInputStream(hdfs.open(new Path(pathSrc)));
    LineReader in = new LineReader(dis);
    insts = InstancesHelper.createInstancesFromMetadata(in);
    in.close();/*w w w  .  ja va2s . c o m*/
    dis.close();
}

From source file:boostingPL.MR.AdaBoostPLTestMapper.java

License:Open Source License

protected void setup(Context context) throws IOException, InterruptedException {
    // classifier file
    Path path = new Path(context.getConfiguration().get("BoostingPL.modelPath") + "/part-r-00000");
    String boostingName = context.getConfiguration().get("BoostingPL.boostingName");
    boostingPL = BoostingPLFactory.createBoostingPL(boostingName, context.getConfiguration(), path);

    // testing dataset metadata
    String pathSrc = context.getConfiguration().get("BoostingPL.metadata");
    FileSystem hdfs = FileSystem.get(context.getConfiguration());
    FSDataInputStream dis = new FSDataInputStream(hdfs.open(new Path(pathSrc)));
    LineReader in = new LineReader(dis);
    insts = InstancesHelper.createInstancesFromMetadata(in);
    in.close();//from  w w  w. j  a va  2 s  . c om
    dis.close();

    try {
        eval = new Evaluation(insts);
    } catch (Exception e) {
        LOG.error("[BoostingPL-Test]: Evaluation init error!");
        e.printStackTrace();
    }
    instanceCounter = context.getCounter("BoostingPL", "Number of instances");
}

From source file:boostingPL.MR.AdaBoostPLTestMapper.java

License:Open Source License

private void output2HDFS(Context context) throws Exception {
    int jobID = context.getJobID().getId();
    int taskID = context.getTaskAttemptID().getTaskID().getId();
    String outputFloder = context.getConfiguration().get("BoostingPL.outputFolder");

    Path path = new Path(outputFloder + "/result_" + jobID + "_m_" + taskID);
    FileSystem hdfs = FileSystem.get(context.getConfiguration());
    FSDataOutputStream outputStream = hdfs.create(path);

    String result = eval.toSummaryString();
    outputStream.write(result.getBytes());
    result = eval.toClassDetailsString();
    outputStream.write(result.getBytes());
    result = eval.toMatrixString();/*  www  .  j  a v a2s.co m*/
    outputStream.write(result.getBytes());
    result = "-----------------------------------------------------------";
    outputStream.write(result.getBytes());

    outputStream.close();
}

From source file:boostingPL.MR.AdaBoostPLTestReducer.java

License:Open Source License

protected void setup(Context context) throws IOException, InterruptedException {
    // classifier file
    Path path = new Path(context.getConfiguration().get("BoostingPL.modelPath") + "/part-r-00000");
    String boostingName = context.getConfiguration().get("BoostingPL.boostingName");
    boostingPL = BoostingPLFactory.createBoostingPL(boostingName, context.getConfiguration(), path);

    // testing dataset metadata
    String pathSrc = context.getConfiguration().get("BoostingPL.metadata");
    FileSystem hdfs = FileSystem.get(context.getConfiguration());
    FSDataInputStream dis = new FSDataInputStream(hdfs.open(new Path(pathSrc)));
    LineReader in = new LineReader(dis);
    insts = InstancesHelper.createInstancesFromMetadata(in);
    in.close();//from  w  w  w.ja  v a 2  s . c om
    dis.close();

    try {
        eval = new Evaluation(insts);
    } catch (Exception e) {
        LOG.error("[BoostingPL-Test]: Evaluation init error!");
        e.printStackTrace();
    }
}

From source file:boostingPL.MR.AdaBoostPLTestReducer.java

License:Open Source License

private void output2HDFS(Context context) throws Exception {
    int jobID = context.getJobID().getId();
    int taskID = context.getTaskAttemptID().getTaskID().getId();
    String outputFloder = context.getConfiguration().get("BoostingPL.outputFolder");

    Path path = new Path(outputFloder + "/result_" + jobID + "_r_" + taskID);
    FileSystem hdfs = FileSystem.get(context.getConfiguration());
    FSDataOutputStream outputStream = hdfs.create(path);

    String result = eval.toSummaryString();
    outputStream.write(result.getBytes());
    result = eval.toClassDetailsString();
    outputStream.write(result.getBytes());
    result = eval.toMatrixString();//ww w .  j a v a  2s .c om
    outputStream.write(result.getBytes());
    result = "-----------------------------------------------------------";
    outputStream.write(result.getBytes());

    outputStream.close();
}

From source file:br.com.lassal.nqueens.grid.job.GenerateSolutions.java

/**
 * NQueens working folder structure /nqueens/board-{x}/partial/solution_X-4
 *
 * @param queensSize//w ww  . ja  v  a 2  s .c o  m
 * @throws IOException
 */
private void setWorkingFolder(int queensSize, Job job) throws IOException {
    Configuration conf = getConf();
    FileSystem fs = FileSystem.get(conf);

    if (fs.isDirectory(new Path("/nqueens/board-" + queensSize + "/final"))) {
        System.exit(0); // ja foi processado anteriormente nao processa de novo
    }

    String lastSolution = null;
    Path partialSolDir = new Path("/nqueens/board-" + queensSize + "/partial/");
    Path inputPath = null;
    Path outputPath = null;

    if (fs.exists(partialSolDir)) {
        RemoteIterator<LocatedFileStatus> dirsFound = fs.listLocatedStatus(partialSolDir);

        while (dirsFound.hasNext()) {
            LocatedFileStatus path = dirsFound.next();
            if (lastSolution == null) {
                lastSolution = path.getPath().getName();
                inputPath = path.getPath();
            } else {
                String currentDir = path.getPath().getName();
                if (lastSolution.compareToIgnoreCase(currentDir) < 0) {
                    lastSolution = currentDir;
                    inputPath = path.getPath();
                }
            }
        }
    }
    int currentSolutionSet = 0;
    if (inputPath == null) {
        inputPath = new Path("/nqueens/board-" + queensSize + "/seed");
        if (!fs.exists(inputPath)) {
            FSDataOutputStream seedFile = fs.create(inputPath, true);
            seedFile.writeBytes(queensSize + "#");
            seedFile.close();
        }
    }
    // Input
    FileInputFormat.addInputPath(job, inputPath);
    job.setInputFormatClass(TextInputFormat.class);

    if (lastSolution != null) {
        String[] solution = lastSolution.split("-");
        if (solution[0].equalsIgnoreCase("solution_" + queensSize)) {
            currentSolutionSet = Integer.parseInt(solution[1]) + 4;

            if (currentSolutionSet >= queensSize) {
                outputPath = new Path("/nqueens/board-" + queensSize + "/final");
            } else {
                outputPath = new Path("/nqueens/board-" + queensSize + "/partial/solution_" + queensSize + "-"
                        + currentSolutionSet);
            }
        }
    } else {
        outputPath = new Path("/nqueens/board-" + queensSize + "/partial/solution_" + queensSize + "-4");
    }

    // Output
    FileOutputFormat.setOutputPath(job, outputPath);
    job.setOutputFormatClass(TextOutputFormat.class);

}