Example usage for org.apache.hadoop.conf Configuration Configuration

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration Configuration.

Prototype

public Configuration()

Source Link

Document

A new configuration.

Usage

From source file:TestIFormatFile.java

License:Open Source License

private static void verify(String filename, int num, int checktime) throws IOException {
    boolean flag = false;
    Random r = new Random();
    Configuration conf = new Configuration();
    IFormatDataFile fdf = new IFormatDataFile(conf);
    fdf.open(filename);//from   w ww.  j a  va2  s .  com
    for (int i = 0; i < checktime; i++) {
        int pos = r.nextInt(num);
        IRecord record = gentestrecord(pos);
        fdf.seek(pos);
        IRecord rec = fdf.next();
        if (!compare(record, rec)) {
            flag = true;
            System.out.println("a error:\t" + pos);
            record.show();
            rec.show();
        }
    }
    fdf.close();
    if (flag) {
        System.out.println("has error");
    } else {
        System.out.println("no error");
    }
}

From source file:TestIFormatFile.java

License:Open Source License

static void readtestshell(String fileName) throws IOException {
    Configuration conf = new Configuration();
    IFormatDataFile fdf = new IFormatDataFile(conf);
    fdf.open(fileName);/*from w  w  w .  jav a  2  s.co  m*/

    IRecord record = new IRecord();
    BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
    while (true) {
        try {
            System.out.println("input param: [seekrowid] [num]");
            StringTokenizer st = new StringTokenizer(br.readLine());
            int line;
            if (st.hasMoreTokens()) {
                line = Integer.parseInt(st.nextToken());
            } else {
                line = -1;
            }
            int num;
            if (st.hasMoreTokens()) {
                num = Integer.parseInt(st.nextToken());

            } else {
                num = 10;
            }

            boolean seek = false;
            long time = System.currentTimeMillis();
            if (line > -1) {
                seek = fdf.seek(line);
            }
            System.out.println("seektime:\t" + (System.currentTimeMillis() - time));
            System.out.println("seek:\t" + seek);

            for (int i = 0; i < num; i++) {
                if (!fdf.next(record)) {
                    break;
                }
                record.show();
            }

        } catch (Exception e) {
            e.printStackTrace();
            try {
                Thread.sleep(500);
            } catch (InterruptedException e1) {
                e1.printStackTrace();
            }
            System.out.println("input again ^_^");
        }
    }
}

From source file:TestIFormatFile.java

License:Open Source License

static void headtest(String fileName) throws IOException {
    Configuration conf = new Configuration();
    IFormatDataFile fdf = new IFormatDataFile(conf);
    fdf.open(fileName);//from  www .j  a v a 2 s  .  c  om
    fdf.next();

}

From source file:TestIFormatFile.java

License:Open Source License

static void randomreadtest(String fileName, int num) throws IOException {
    Configuration conf = new Configuration();
    IFormatDataFile fdf = new IFormatDataFile(conf);
    fdf.open(fileName);//ww w.ja  va  2  s . c  o  m

    IRecord record = new IRecord();
    int size = fdf.recnum();
    if (size == -1) {
        System.out.println("can not seek");
        return;
    }
    for (int i = 0; i < num; i++) {
        int seekpos = (int) (Math.random() * size);
        long time = System.currentTimeMillis();
        fdf.seek(seekpos);
        fdf.next(record);
        System.out.println((System.currentTimeMillis() - time));
    }
}

From source file:TestIFormatFile.java

License:Open Source License

static void seekfvtest(String fileName, int start, int step, int num) throws IOException {
    Configuration conf = new Configuration();
    IFormatDataFile fdf = new IFormatDataFile(conf);
    fdf.open(fileName);/*  w ww  .j a v  a  2  s. c  o m*/

    IRecord record = new IRecord();
    for (int i = 0; i < num; i++) {
        IFieldValue fv = new IFieldValue((short) 345);
        if (fdf.seek(fv)) {
            fdf.next(record);
            record.show();
            fdf.next(record);
            record.show();
            fdf.next(record);
            record.show();
            fdf.next(record);
            record.show();
            fdf.next(record);
            record.show();
            fdf.next(record);
            record.show();
            fdf.next(record);
            record.show();
            fdf.next(record);
            record.show();
            fdf.next(record);
            record.show();
        }
    }

}

From source file:TestIFormatFile.java

License:Open Source License

static void test1() throws IOException {
    String fileName = "/testtesttest2";

    Configuration conf = new Configuration();
    IFormatDataFile fdf = new IFormatDataFile(conf);
    IHead head = new IHead();

    IFieldMap fieldMap = new IFieldMap();
    fieldMap.addFieldType(new IFieldType.IFieldByteType());
    fieldMap.addFieldType(new IFieldType.IFieldShortType());
    fieldMap.addFieldType(new IFieldType.IFieldIntType());
    fieldMap.addFieldType(new IFieldType.IFieldLongType());
    fieldMap.addFieldType(new IFieldType.IFieldFloatType());
    fieldMap.addFieldType(new IFieldType.IFieldDoubleType());

    head.setFieldMap(fieldMap);/*from  w w w.ja  va2  s  .  c  o  m*/

    head.setPrimaryIndex((short) 2);

    IUserDefinedHeadInfo udi = new IUserDefinedHeadInfo();
    udi.addInfo(0, fileName);

    fdf.create(fileName, head);
    IRecord record = new IRecord();
    int i = 100;
    record.addFieldValue(new IFieldValue((byte) i));
    record.addFieldValue(new IFieldValue((short) (2 * i)));
    record.addFieldValue(new IFieldValue(3 * i));
    record.addFieldValue(new IFieldValue((long) 4 * i));

    fdf.addRecord(record);
    fdf.close();

}

From source file:TestComplexColumnInSchema.java

License:Apache License

@BeforeClass
public static void setup() throws IOException {
    Configuration conf = new Configuration();

    footer = ParquetFileReader.readFooter(conf, new Path(path));
}

From source file:LobFileStressTest.java

License:Apache License

public LobFileStressTest() {
    conf = new Configuration();
    conf.set("fs.default.name", "file:///");
    allPassed = true;
}

From source file:Top20AZRestaurants.java

public static void main(String[] args) throws Exception {
    int exitCode = ToolRunner.run(new Configuration(), new Top20AZRestaurants(), args);
    System.exit(exitCode);//www .jav  a 2  s.c  o  m
}

From source file:WordCountB.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
        System.err.println("Usage: wordcount <in> [<in>...] <out>");
        System.exit(2);//from w w w .j a v  a2  s . c o  m
    }
    Job job = Job.getInstance(conf, "word count");
    job.setJarByClass(WordCountB.class);
    job.setMapperClass(TokenizerMapper.class);
    // Setup the Combiner
    job.setCombinerClass(IntSumReducer.class);
    // Setup the Partitioner
    job.setPartitionerClass(Letterpartitioner.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    for (int i = 0; i < otherArgs.length - 1; ++i) {
        FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
    }
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}