Example usage for org.apache.hadoop.conf Configuration Configuration

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration Configuration.

Prototype

public Configuration()

Source Link

Document

A new configuration.

Usage

From source file:audr.text.utils.FileUtils.java

License:Open Source License

/**
 * create index dirs for each text category
 * @param root/*w w w. j  a v a  2 s.c  o  m*/
 * @throws IOException 
 */
public static void makeIndexDirs() throws IOException {
    FileSystem fs = FileSystem.get(new Configuration());
    for (int i = 0; i < TextCategoryFields.TEXT_CATEGOTIES_ENUM.length; ++i) {
        String oriDir = Constants.INPUT_PATH.replace("%Category%", TextCategoryFields.TEXT_CATEGOTIES_ENUM[i]);
        String lfDir = Constants.INPUT_PATH_LF.replace("%Category%",
                TextCategoryFields.TEXT_CATEGOTIES_ENUM[i]);
        FileSystem.mkdirs(fs, new Path(oriDir), FsPermission.getDefault());
        FileSystem.mkdirs(fs, new Path(lfDir), FsPermission.getDefault());

        for (int j = 0; j < Constants.INDEX_SHARDS.length; ++j) {
            String indexDir = Constants.INDEX_SHARDS[j].replace("%Category%",
                    TextCategoryFields.TEXT_CATEGOTIES_ENUM[i]);
            FileSystem.mkdirs(fs, new Path(indexDir), FsPermission.getDefault());
        }
    }
}

From source file:averageprocessingtimesbytype.AverageProcessingTimesByType.java

public static void main(String[] args) throws Exception {
    int res = ToolRunner.run(new Configuration(), new AverageProcessingTimesByType(), args);
    System.exit(res);/*  ww w  .j av  a 2 s  . c o  m*/
}

From source file:AverageProj.AveragePrice.java

public static void main(String[] args) throws Exception {
    int res = ToolRunner.run(new Configuration(), new AveragePrice(), args);
    System.exit(res);//  w w  w  .  j a  v  a 2s . c  om
}

From source file:averagerating_youtube.AverageRating_Youtube.java

public static void main(String[] args) throws Exception {
    // TODO code application logic here
    int exitCode = ToolRunner.run(new Configuration(), new AverageRating_Youtube(), args);
    System.exit(exitCode);/* ww w  .ja  va  2 s  . c o  m*/
}

From source file:avro.HadoopAvro.java

License:Open Source License

private JobConf createJobConfig() throws IOException {
    Path inputPath = new Path(INPUT_PATH);
    Path outputPath = new Path(OUTPUT_PATH);

    FileSystem.get(new Configuration()).delete(outputPath, true);

    JobConf jobConfig = new JobConf();
    jobConfig.setInputFormat(AvroInputFormat.class);
    jobConfig.setOutputFormat(AvroOutputFormat.class);
    AvroOutputFormat.setOutputPath(jobConfig, outputPath);
    AvroInputFormat.addInputPath(jobConfig, inputPath);
    jobConfig.set(AvroJob.OUTPUT_SCHEMA, User.SCHEMA.toString());
    jobConfig.set(AvroJob.INPUT_SCHEMA, User.SCHEMA.toString());
    return jobConfig;
}

From source file:avro.HadoopAvro.java

License:Open Source License

private void createAvroFile() throws IOException {
    Path inputPath = new Path(INPUT_PATH);
    FileSystem fs = FileSystem.get(new Configuration());
    fs.delete(inputPath, true);/*from  ww w.  jav  a  2  s . c om*/

    DataFileWriter<User> fileWriter = new DataFileWriter<>(new GenericDatumWriter<User>(User.SCHEMA));

    fileWriter.create(User.SCHEMA, fs.create(new Path(inputPath, "file.avro")));
    IntStream.range(0, 100).mapToObj(i -> new User("name" + i, "pass" + i, i, i % 2 == 0))
            .forEach(user -> Util.uncheckRun(() -> fileWriter.append(user)));
    fileWriter.close();
    fs.close();
}

From source file:avro.mr.MapReduceAvroWordCount.java

License:Apache License

public static void main(String[] args) throws Exception {
    int res = ToolRunner.run(new Configuration(), new MapReduceAvroWordCount(), args);
    System.exit(res);/*  ww  w . ja  va  2 s.  c  o m*/
}

From source file:azkaban.common.web.HdfsSequenceFileViewer.java

License:Apache License

public boolean canReadFile(FileSystem fs, Path file) {
    boolean result = false;
    try {/* w w  w .  j a v a  2  s  . co m*/
        SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, new Configuration());
        result = canReadFile(reader);
        reader.close();
    } catch (IOException e) {
        return false;
    }

    return result;
}

From source file:azkaban.common.web.HdfsSequenceFileViewer.java

License:Apache License

public void displayFile(FileSystem fs, Path file, OutputStream outputStream, int startLine, int endLine)
        throws IOException {
    SequenceFile.Reader reader = null;
    PrintWriter writer = new PrintWriter(outputStream);
    try {//from  w  ww  .j  a va 2s .c om
        reader = new SequenceFile.Reader(fs, file, new Configuration());
        displaySequenceFile(reader, writer, startLine, endLine);
    } catch (IOException e) {
        writer.write("Error opening sequence file " + e);
    } finally {
        if (reader != null) {
            reader.close();
        }
    }
}

From source file:azkaban.jobtype.connectors.HdfsToTeradataJobRunnerMain.java

License:Apache License

public HdfsToTeradataJobRunnerMain() throws FileNotFoundException, IOException {
    _logger = JobUtils.initJobLogger();//w  w w .j a v a2s  .  c o m
    _jobProps = HadoopSecureWrapperUtils.loadAzkabanProps();

    Props props = new Props(null, _jobProps);
    HadoopConfigurationInjector.injectResources(props);
    UserGroupInformation.setConfiguration(new Configuration());

    _params = TdchParameters.builder().mrParams(_jobProps.getProperty(TdchConstants.HADOOP_CONFIG_KEY))
            .libJars(props.getString(TdchConstants.LIB_JARS_KEY))
            .tdJdbcClassName(TdchConstants.TERADATA_JDBCDRIVER_CLASSNAME)
            .teradataHostname(props.getString(TdchConstants.TD_HOSTNAME_KEY))
            .fileFormat(_jobProps.getProperty(TdchConstants.HDFS_FILE_FORMAT_KEY))
            .fieldSeparator(_jobProps.getProperty(TdchConstants.HDFS_FIELD_SEPARATOR_KEY))
            .jobType(TdchConstants.TDCH_JOB_TYPE).userName(props.getString(TdchConstants.TD_USERID_KEY))
            .credentialName(String.format(TdchConstants.TD_WALLET_FORMAT,
                    props.getString(TdchConstants.TD_CREDENTIAL_NAME)))
            .avroSchemaPath(_jobProps.getProperty(TdchConstants.AVRO_SCHEMA_PATH_KEY))
            .avroSchemaInline(_jobProps.getProperty(TdchConstants.AVRO_SCHEMA_INLINE_KEY))
            .sourceHdfsPath(props.getString(TdchConstants.SOURCE_HDFS_PATH_KEY))
            .targetTdTableName(props.getString(TdchConstants.TARGET_TD_TABLE_NAME_KEY))
            .tdInsertMethod(_jobProps.getProperty(TdchConstants.TD_INSERT_METHOD_KEY))
            .numMapper(DEFAULT_NO_MAPPERS).build();
}