Example usage for org.apache.hadoop.conf Configuration Configuration

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration Configuration.

Prototype

public Configuration() 

Source Link

Document

A new configuration.

Usage

From source file:TestIndexServer.java

License:Open Source License

@Override
protected void setUp() throws Exception {
    super.setUp();
    conf = new Configuration();
    fs = FileSystem.get(conf);//  www . j  a  v  a2 s  . com
}

---

From source file:TestIndexServer.java

License:Open Source License

public void testServerMergeIndex() throws Exception {
    Configuration conf = new Configuration();
    conf.set("se.indexer.IPM_merge_interval", "5000");
    IndexServer server = new IndexServer(conf);
    server.testmode = true;//from w w  w. j  ava 2  s  .com
    server.start();
    ArrayList<IndexItemStatus> itemstatuss = new ArrayList<IndexItemStatus>();
    for (int k = 0; k < 0; k++) {
        while (true) {
            itemstatuss.clear();
            File file = new File("indexconf");
            DataInputStream dis = new DataInputStream(new FileInputStream(file));
            int num = dis.readInt();
            for (int i = 0; i < num; i++) {
                IndexItemStatus status = new IndexItemStatus();
                status.read(dis);
                itemstatuss.add(status);
            }
            dis.close();
            if (itemstatuss.get(0).status == 2) {
                break;
            }
            if (itemstatuss.get(0).status < 0) {
                server.close();
                assertTrue(false);
            }
            Thread.sleep(3000);
        }
        UtilIndexStorage.writeFDF(status.gettablelocation(), 3, 10000, (short) -1, false, false, true, false);
        while (true) {
            itemstatuss.clear();
            File file = new File("indexconf");
            DataInputStream dis = new DataInputStream(new FileInputStream(file));
            int num = dis.readInt();
            for (int i = 0; i < num; i++) {
                IndexItemStatus status = new IndexItemStatus();
                status.read(dis);
                itemstatuss.add(status);
            }
            dis.close();
            if (itemstatuss.get(0).status == 1) {
                break;
            }
            Thread.sleep(3000);
        }
    }

    Thread.sleep(15000);
    while (true) {
        itemstatuss.clear();
        File file = new File("indexconf");
        DataInputStream dis = new DataInputStream(new FileInputStream(file));
        int num = dis.readInt();
        for (int i = 0; i < num; i++) {
            IndexItemStatus status = new IndexItemStatus();
            status.read(dis);
            itemstatuss.add(status);
        }
        dis.close();
        if (itemstatuss.get(0).status == 2) {
            break;
        }
        Thread.sleep(3000);
    }

    FileStatus[] fss = fs.listStatus(new Path(itemstatuss.get(0).indexlocation + "/nopart"));
    assertEquals(fss.length, 1);

    server.close();
}

---

From source file:ColumnStroageStabilityTest.java

License:Open Source License

public static void doWrite(int count) throws Exception {
    Configuration conf = new Configuration();

    System.out.println("write byte file:" + byteFileName);
    FieldMap byteFieldMap = new FieldMap();
    byteFieldMap.addField(new Field(ConstVar.FieldType_Byte, ConstVar.Sizeof_Byte, (short) 0));

    Head byteHead = new Head();
    byteHead.setFieldMap(byteFieldMap);//from   w  w w  . j  a v  a  2  s  .c  o  m

    FormatDataFile byteFD = new FormatDataFile(conf);
    byteFD.create(byteFileName, byteHead);

    System.out.println("write short file:" + shortFileName);
    FieldMap shortFieldMap = new FieldMap();
    shortFieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 1));

    Head shortHead = new Head();
    shortHead.setFieldMap(shortFieldMap);

    FormatDataFile shortFD = new FormatDataFile(conf);
    shortFD.create(shortFileName, shortHead);

    System.out.println("write int file:" + intFileName);
    FieldMap intFieldMap = new FieldMap();
    intFieldMap.addField(new Field(ConstVar.FieldType_Int, ConstVar.Sizeof_Int, (short) 2));

    Head intHead = new Head();
    intHead.setFieldMap(intFieldMap);

    FormatDataFile intFD = new FormatDataFile(conf);
    intFD.create(intFileName, intHead);

    System.out.println("write long file:" + longFileName);
    FieldMap longFieldMap = new FieldMap();
    longFieldMap.addField(new Field(ConstVar.FieldType_Long, ConstVar.Sizeof_Long, (short) 3));

    Head longHead = new Head();
    longHead.setFieldMap(longFieldMap);

    FormatDataFile longFD = new FormatDataFile(conf);
    longFD.create(longFileName, longHead);

    System.out.println("write float file:" + floatFileName);
    FieldMap floatFieldMap = new FieldMap();
    floatFieldMap.addField(new Field(ConstVar.FieldType_Float, ConstVar.Sizeof_Float, (short) 4));

    Head floatHead = new Head();
    floatHead.setFieldMap(floatFieldMap);

    FormatDataFile floatFD = new FormatDataFile(conf);
    floatFD.create(floatFileName, floatHead);

    System.out.println("write double file:" + doubleFileName);
    FieldMap doubleFieldMap = new FieldMap();
    doubleFieldMap.addField(new Field(ConstVar.FieldType_Double, ConstVar.Sizeof_Double, (short) 5));

    Head doubleHead = new Head();
    doubleHead.setFieldMap(doubleFieldMap);

    FormatDataFile doubleFD = new FormatDataFile(conf);
    doubleFD.create(doubleFileName, doubleHead);

    System.out.println("write int file:" + stringFileName);
    FieldMap strFieldMap = new FieldMap();
    strFieldMap.addField(new Field(ConstVar.FieldType_String, 0, (short) 6));

    Head strHead = new Head();
    strHead.setFieldMap(strFieldMap);

    FormatDataFile strFD = new FormatDataFile(conf);
    strFD.create(stringFileName, strHead);

    System.out.println("write int file:" + multiFileNameString);
    FieldMap multiFieldMap = new FieldMap();
    multiFieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 7));
    multiFieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 8));
    multiFieldMap.addField(new Field(ConstVar.FieldType_Short, ConstVar.Sizeof_Short, (short) 9));

    Head multiHead = new Head();
    multiHead.setFieldMap(multiFieldMap);

    FormatDataFile multiFD = new FormatDataFile(conf);
    multiFD.create(multiFileNameString, multiHead);

    long begin = System.currentTimeMillis();
    for (int i = 0; i < count; i++) {
        createProjectByte(byteFD);
        createProjectShort(shortFD);
        createProjectInt(intFD);
        createProjectLong(longFD);
        createProjectFloat(floatFD);
        createProjectDouble(doubleFD);

        if (i % 1000000 == 0) {
            long end = System.currentTimeMillis();
            System.out.println(i + " record ok, delay:" + (end - begin) / 1000 + " s");
        }

    }

    byteFD.close();
    shortFD.close();
    intFD.close();
    longFD.close();
    floatFD.close();
    doubleFD.close();
    strFD.close();
    multiFD.close();

    long end = System.currentTimeMillis();
    System.out.println(count + " record over(column), delay:" + (end - begin) / 1000 + " s");

}

---

From source file:ColumnStroageStabilityTest.java

License:Open Source License

static void doReadRand(int count) {
    try {/*w ww.  j a  v a 2 s  . c  o  m*/

        Path path = new Path(fullPrefix);
        ArrayList<Short> vector = new ArrayList<Short>(10);
        vector.add((short) 3);
        vector.add((short) 6);
        vector.add((short) 4);
        vector.add((short) 2);
        vector.add((short) 0);
        vector.add((short) 5);
        vector.add((short) 1);

        Configuration conf = new Configuration();
        ColumnStorageClient client = new ColumnStorageClient(path, vector, conf);
        long begin = System.currentTimeMillis();

        for (int i = 0; i < count; i++) {
            int rand = (int) (Math.random() * count);
            Record record = client.getRecordByLine(rand);
            /*
             if (record == null)
             {
            String string = "record no:" + i + " return null";
            out.write(string.getBytes());
             }
                     
                    
             if (!judgeRecord(record))
             {
            String string = "record no:" + i + " value error";
            out.write(string.getBytes());
             }
             */
            if (i % (100 * 10000) == 0) {
                String string = "read seq " + i + " record, delay:"
                        + ((System.currentTimeMillis() - begin) / 1000) + " s \n";
                System.out.println(string);
            }
        }

        long end = System.currentTimeMillis();
        String string = "Read Seq over, count:" + count + ", delay:" + (long) ((end - begin) / 1000) + " s";
        System.out.println(string);
    } catch (IOException e) {
        e.printStackTrace();
        System.out.println("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        System.out.println("get exception:" + e.getMessage());
    }

}

---

From source file:ColumnStroageStabilityTest.java

License:Open Source License

static void doReadSeq(int count) {
    try {// ww  w  .j  a v  a2  s .c  o  m

        Path path = new Path(prefix);
        ArrayList<Short> vector = new ArrayList<Short>(10);

        File file = new File("testReadMassRecord.result");
        FileOutputStream out = new FileOutputStream(file);

        for (int i = 0; i < idxs.length; i++) {
            vector.add(Short.valueOf(idxs[i]));
        }

        Configuration conf = new Configuration();
        ColumnStorageClient client = new ColumnStorageClient(path, vector, conf);
        long begin = System.currentTimeMillis();

        for (int i = 0; i < count; i++) {
            Record record = client.getRecordByLine(i);

            if (record == null) {
                String string = "record no:" + i + " return null \n";
                out.write(string.getBytes());
            }

            Error error = new Error();
            if (!judgeRecord(record, error)) {
                String string = "record no:" + i + ": " + error.msg + "\n";
                out.write(string.getBytes());
            }

            if (i % (100 * 10000) == 0) {
                String string = "read seq " + i + " record, delay:"
                        + ((System.currentTimeMillis() - begin) / 1000) + " s ";
                out.write(string.getBytes());
                out.write(new String("\n").getBytes());
            }
        }

        client.close();

        long end = System.currentTimeMillis();
        String string = "Read Seq over, count:" + count + ", delay:" + (long) ((end - begin) / 1000) + " s \n";
        out.write(string.getBytes());
        out.close();
        System.out.println(string);
    } catch (IOException e) {
        e.printStackTrace();
        System.out.println("get IOException:" + e.getMessage());
    } catch (Exception e) {
        e.printStackTrace();
        System.out.println("get exception:" + e.getMessage());
    }

}

---

From source file:TweetCategorizer.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    // conf.addResource(new Path("../../env_vars"));

    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 2) {
        System.err.println("Usage: TweetCategorizer <in> <out>");
        System.exit(2);//w  w  w  .j a va 2s  . c  o m
    }

    // ----------------------------------------------------------
    //         READ FILTER FILE
    // ----------------------------------------------------------
    // Path pt=new Path("hdfs://pathTofile");
    //Path pt = new Path("../www/hfilters.json");
    String l;
    String line = "";
    //FileSystem fs = FileSystem.get(conf);
    BufferedReader br = new BufferedReader(new FileReader("../www/json/filters.json"));

    try {
        //BufferedReader br = new BufferedReader(new FileReader(fs.open(pt)));

        while ((l = br.readLine()) != null) {
            line += l;
            //System.out.println(line);
        }

    } finally {
        // you should close out the BufferedReader
        br.close();
    }
    // ----------------------------------------------------------
    //         PARSE JSON
    //http://stackoverflow.com/questions/6697147/json-iterate-through-jsonarray
    //http://juliusdavies.ca/json-simple-1.1.1-javadocs/org/json/simple/JSONObject.html
    // ----------------------------------------------------------
    JSONParser parser = new JSONParser();
    JSONObject jsonObject = (JSONObject) parser.parse(line);

    Set<String> filters = jsonObject.keySet();

    // inside each object there is a "name" field, get value and add to keyword_list
    for (String i : filters) {
        JSONObject objects = (JSONObject) jsonObject.get(i);
        String keyword = ((String) objects.get("name")).toLowerCase();
        TokenizerMapper.keyname_list.add(i);
        TokenizerMapper.keyword_list.add(keyword);
    }
    // ----------------------------------------------------------

    Job job = new Job(conf, "categorize tweets");
    job.setJarByClass(TweetCategorizer.class);
    job.setMapperClass(TokenizerMapper.class);
    // job.setCombinerClass(IntSumReducer.class);
    // job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

---

From source file:WordLines.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 3) {
        System.err.println("Usage: wordlines <in> [<in>...] <SearchTerm> <out>");
        System.exit(2);//w ww  .jav a2s. c om
    }
    conf.set("searchWord", otherArgs[otherArgs.length - 2]);
    Job job = new Job(conf, "word lines");
    job.setJarByClass(WordLines.class);
    job.setMapperClass(TokenizerMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    for (int i = 0; i < otherArgs.length - 2; ++i) {
        FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
    }
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

---

From source file:DataJoinJob.java

License:Apache License

public static JobConf createDataJoinJob(String args[]) throws IOException {

    String inputDir = args[0];/* www. j a va  2  s. co m*/
    String outputDir = args[1];
    Class inputFormat = SequenceFileInputFormat.class;
    if (args[2].compareToIgnoreCase("text") != 0) {
        System.out.println("Using SequenceFileInputFormat: " + args[2]);
    } else {
        System.out.println("Using TextInputFormat: " + args[2]);
        inputFormat = TextInputFormat.class;
    }
    int numOfReducers = Integer.parseInt(args[3]);
    Class mapper = getClassByName(args[4]);
    Class reducer = getClassByName(args[5]);
    Class mapoutputValueClass = getClassByName(args[6]);
    Class outputFormat = TextOutputFormat.class;
    Class outputValueClass = Text.class;
    if (args[7].compareToIgnoreCase("text") != 0) {
        System.out.println("Using SequenceFileOutputFormat: " + args[7]);
        outputFormat = SequenceFileOutputFormat.class;
        outputValueClass = getClassByName(args[7]);
    } else {
        System.out.println("Using TextOutputFormat: " + args[7]);
    }
    long maxNumOfValuesPerGroup = 100;
    String jobName = "";
    if (args.length > 8) {
        maxNumOfValuesPerGroup = Long.parseLong(args[8]);
    }
    if (args.length > 9) {
        jobName = args[9];
    }
    Configuration defaults = new Configuration();
    JobConf job = new JobConf(defaults, DataJoinJob.class);
    job.setJobName("DataJoinJob: " + jobName);

    FileSystem fs = FileSystem.get(defaults);
    fs.delete(new Path(outputDir));
    FileInputFormat.setInputPaths(job, inputDir);

    job.setInputFormat(inputFormat);

    job.setMapperClass(mapper);
    FileOutputFormat.setOutputPath(job, new Path(outputDir));
    job.setOutputFormat(outputFormat);
    SequenceFileOutputFormat.setOutputCompressionType(job, SequenceFile.CompressionType.BLOCK);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(mapoutputValueClass);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(outputValueClass);
    job.setReducerClass(reducer);

    job.setNumMapTasks(1);
    job.setNumReduceTasks(numOfReducers);
    job.setLong("datajoin.maxNumOfValuesPerGroup", maxNumOfValuesPerGroup);
    return job;
}

---

From source file:HDFSRandomAccessFile.java

License:Apache License

public HDFSRandomAccessFile(String fileSystemURI, String location, int bufferSize) throws IOException {
    super(bufferSize);
    fsURI = URI.create(fileSystemURI);
    filePath = new Path(location);
    this.location = location;
    if (debugLeaks) {
        openFiles.add(location);/*from  w ww  .j a  v a2  s.  c om*/
    }

    FileSystem fs = FileSystem.get(fsURI, new Configuration());
    hfile = fs.open(filePath);

    fileStatus = fs.getFileStatus(filePath);
}

---

From source file:WordCountSplitTest.java

License:Apache License

private final static void test(boolean use_shards, boolean use_chunks, Boolean slaveok) throws Exception {
    did_start = false;/*  w w w  .j ava 2 s  .c  o m*/
    final Configuration conf = new Configuration();
    MongoConfigUtil.setInputURI(conf, "mongodb://localhost:30000/test.lines");
    conf.setBoolean(MongoConfigUtil.SPLITS_USE_SHARDS, use_shards);
    conf.setBoolean(MongoConfigUtil.SPLITS_USE_CHUNKS, use_chunks);
    String output_table = null;
    if (use_chunks) {
        if (use_shards)
            output_table = "with_shards_and_chunks";
        else
            output_table = "with_chunks";
    } else {
        if (use_shards)
            output_table = "with_shards";
        else
            output_table = "no_splits";
    }
    if (slaveok != null) {
        output_table += "_" + slaveok;
    }
    MongoConfigUtil.setOutputURI(conf, "mongodb://localhost:30000/test." + output_table);
    System.out.println("Conf: " + conf);

    final Job job = new Job(conf, "word count " + output_table);

    job.setJarByClass(WordCountSplitTest.class);

    job.setMapperClass(TokenizerMapper.class);

    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setInputFormatClass(MongoInputFormat.class);
    job.setOutputFormatClass(MongoOutputFormat.class);

    final long start = System.currentTimeMillis();
    System.out.println(" ----------------------- running test " + output_table + " --------------------");
    try {
        boolean result = job.waitForCompletion(true);
        System.out.println("job.waitForCompletion( true ) returned " + result);
    } catch (Exception e) {
        System.out.println("job.waitForCompletion( true ) threw Exception");
        e.printStackTrace();
    }
    final long end = System.currentTimeMillis();
    final float seconds = ((float) (end - start)) / 1000;
    java.text.NumberFormat nf = java.text.NumberFormat.getInstance();
    nf.setMaximumFractionDigits(3);
    System.out.println("finished run in " + nf.format(seconds) + " seconds");

    com.mongodb.Mongo m = new com.mongodb.Mongo(
            new com.mongodb.MongoURI("mongodb://localhost:30000/?slaveok=true"));
    com.mongodb.DB db = m.getDB("test");
    com.mongodb.DBCollection coll = db.getCollection(output_table);
    com.mongodb.BasicDBObject query = new com.mongodb.BasicDBObject();
    query.put("_id", "the");
    com.mongodb.DBCursor cur = coll.find(query);
    if (!cur.hasNext())
        System.out.println("FAILURE: could not find count of \'the\'");
    else
        System.out.println("'the' count: " + cur.next());

    //        if (! result)
    //           System.exit(  1 );
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »