Example usage for org.apache.hadoop.conf Configuration addResource

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration addResource.

Prototype

public void addResource(Configuration conf) 

Source Link

Document

Add a configuration resource.

Usage

From source file:org.apache.ivory.retention.FeedEvictor.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    Path confPath = new Path("file:///" + System.getProperty("oozie.action.conf.xml"));

    LOG.info(confPath + " found ? " + confPath.getFileSystem(conf).exists(confPath));
    conf.addResource(confPath);
    int ret = ToolRunner.run(conf, new FeedEvictor(), args);
    if (ret != 0) {
        throw new Exception("Unable to perform eviction action args: " + Arrays.toString(args));
    }/*from  w  w  w.  jav  a2  s.c o  m*/
}

---

From source file:org.apache.ivory.util.OozieUtils.java

License:Apache License

public static Properties toProperties(String properties) {
    Configuration conf = new Configuration(false);
    conf.addResource(new ByteArrayInputStream(properties.getBytes()));
    Properties jobprops = new Properties();
    for (Map.Entry<String, String> entry : conf) {
        jobprops.put(entry.getKey(), entry.getValue());
    }/*from  w  w  w.  j  a  v a2  s. co  m*/
    return jobprops;
}

---

From source file:org.apache.kylin.source.hive.cardinality.HiveColumnCardinalityJob.java

License:Apache License

@Override
public int run(String[] args) throws Exception {

    Options options = new Options();

    options.addOption(OPTION_TABLE);//w  ww. j  a  v a2 s.  c  om
    options.addOption(OPTION_OUTPUT_PATH);

    parseOptions(options, args);

    // start job
    String jobName = JOB_TITLE + getOptionsAsString();
    logger.info("Starting: " + jobName);
    Configuration conf = getConf();

    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    JobEngineConfig jobEngineConfig = new JobEngineConfig(kylinConfig);
    conf.addResource(new Path(jobEngineConfig.getHadoopJobConfFilePath(null)));

    job = Job.getInstance(conf, jobName);

    setJobClasspath(job, kylinConfig);

    String table = getOptionValue(OPTION_TABLE);
    job.getConfiguration().set(BatchConstants.CFG_TABLE_NAME, table);

    Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
    FileOutputFormat.setOutputPath(job, output);
    job.getConfiguration().set("dfs.block.size", "67108864");
    job.getConfiguration().set("mapreduce.output.fileoutputformat.compress", "false");

    // Mapper
    IMRTableInputFormat tableInputFormat = MRUtil.getTableInputFormat(table);
    tableInputFormat.configureJob(job);

    job.setMapperClass(ColumnCardinalityMapper.class);
    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(BytesWritable.class);

    // Reducer - only one
    job.setReducerClass(ColumnCardinalityReducer.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(LongWritable.class);
    job.setNumReduceTasks(1);

    this.deletePath(job.getConfiguration(), output);

    logger.info("Going to submit HiveColumnCardinalityJob for table '" + table + "'");

    TableDesc tableDesc = MetadataManager.getInstance(kylinConfig).getTableDesc(table);
    attachTableMetadata(tableDesc, job.getConfiguration());
    int result = waitForCompletion(job);

    return result;
}

---

From source file:org.apache.kylin.source.kafka.config.KafkaConsumerProperties.java

License:Apache License

private Properties loadKafkaConsumerProperties() {
    File propFile = getKafkaConsumerFile();
    if (propFile == null || !propFile.exists()) {
        logger.warn("fail to locate " + KAFKA_CONSUMER_FILE + ", use empty kafka consumer properties");
        return new Properties();
    }//from   w  w w .  j  a v  a2 s .c om
    Properties properties = new Properties();
    try {
        FileInputStream is = new FileInputStream(propFile);
        Configuration conf = new Configuration();
        conf.addResource(is);
        properties.putAll(extractKafkaConfigToProperties(conf));
        IOUtils.closeQuietly(is);

        File propOverrideFile = new File(propFile.getParentFile(), propFile.getName() + ".override");
        if (propOverrideFile.exists()) {
            FileInputStream ois = new FileInputStream(propOverrideFile);
            Configuration oconf = new Configuration();
            oconf.addResource(ois);
            properties.putAll(extractKafkaConfigToProperties(oconf));
            IOUtils.closeQuietly(ois);
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }

    return properties;
}

---

From source file:org.apache.kylin.storage.hbase.HBaseConnection.java

License:Apache License

public static void addHBaseClusterNNHAConfiguration(Configuration conf) {
    String hdfsConfigFile = KylinConfig.getInstanceFromEnv().getHBaseClusterHDFSConfigFile();
    if (hdfsConfigFile == null || hdfsConfigFile.isEmpty()) {
        return;//from   ww  w.  j a  va  2 s  .c o m
    }
    Configuration hdfsConf = new Configuration(false);
    hdfsConf.addResource(hdfsConfigFile);
    Collection<String> nameServices = hdfsConf.getTrimmedStringCollection(DFSConfigKeys.DFS_NAMESERVICES);
    Collection<String> mainNameServices = conf.getTrimmedStringCollection(DFSConfigKeys.DFS_NAMESERVICES);
    for (String serviceId : nameServices) {
        mainNameServices.add(serviceId);

        String serviceConfKey = DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + serviceId;
        String proxyConfKey = DFSConfigKeys.DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "." + serviceId;
        conf.set(serviceConfKey, hdfsConf.get(serviceConfKey, ""));
        conf.set(proxyConfKey, hdfsConf.get(proxyConfKey, ""));

        Collection<String> nameNodes = hdfsConf.getTrimmedStringCollection(serviceConfKey);
        for (String nameNode : nameNodes) {
            String rpcConfKey = DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + serviceId + "." + nameNode;
            conf.set(rpcConfKey, hdfsConf.get(rpcConfKey, ""));
        }
    }
    conf.setStrings(DFSConfigKeys.DFS_NAMESERVICES, mainNameServices.toArray(new String[0]));
    // See YARN-3021, instruct RM skip renew token of hbase cluster name services
    conf.setStrings(JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE, nameServices.toArray(new String[0]));
}

---

From source file:org.apache.kylin.storage.hbase.steps.SparkCubeHFile.java

License:Apache License

@Override
protected void execute(OptionsHelper optionsHelper) throws Exception {
    final String metaUrl = optionsHelper.getOptionValue(OPTION_META_URL);
    final String inputPath = optionsHelper.getOptionValue(OPTION_INPUT_PATH);
    final String cubeName = optionsHelper.getOptionValue(OPTION_CUBE_NAME);
    final String segmentId = optionsHelper.getOptionValue(OPTION_SEGMENT_ID);
    final String outputPath = optionsHelper.getOptionValue(OPTION_OUTPUT_PATH);
    final Path partitionFilePath = new Path(optionsHelper.getOptionValue(OPTION_PARTITION_FILE_PATH));
    final String hbaseConfFile = optionsHelper.getOptionValue(AbstractHadoopJob.OPTION_HBASE_CONF_PATH);
    final String counterPath = optionsHelper.getOptionValue(OPTION_COUNTER_PATH);

    Class[] kryoClassArray = new Class[] { Class.forName("scala.reflect.ClassTag$$anon$1"),
            KeyValueCreator.class, KeyValue.class, RowKeyWritable.class };

    SparkConf conf = new SparkConf().setAppName("Converting HFile for:" + cubeName + " segment " + segmentId);
    //serialization conf
    conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
    conf.set("spark.kryo.registrator", "org.apache.kylin.engine.spark.KylinKryoRegistrator");
    conf.set("spark.kryo.registrationRequired", "true").registerKryoClasses(kryoClassArray);

    KylinSparkJobListener jobListener = new KylinSparkJobListener();
    try (JavaSparkContext sc = new JavaSparkContext(conf)) {
        sc.sc().addSparkListener(jobListener);
        final FileSystem fs = partitionFilePath.getFileSystem(sc.hadoopConfiguration());
        if (!fs.exists(partitionFilePath)) {
            throw new IllegalArgumentException("File not exist: " + partitionFilePath.toString());
        }/* www.  ja va  2 s .  c o  m*/

        HadoopUtil.deletePath(sc.hadoopConfiguration(), new Path(outputPath));
        final SerializableConfiguration sConf = new SerializableConfiguration(sc.hadoopConfiguration());

        final KylinConfig envConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(sConf, metaUrl);

        final CubeInstance cubeInstance = CubeManager.getInstance(envConfig).getCube(cubeName);
        final CubeDesc cubeDesc = cubeInstance.getDescriptor();
        final CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);

        final MeasureCodec inputCodec = new MeasureCodec(cubeDesc.getMeasures());
        final List<KeyValueCreator> keyValueCreators = Lists.newArrayList();

        for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
            for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
                keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc));
            }
        }

        final int cfNum = keyValueCreators.size();
        final boolean quickPath = (keyValueCreators.size() == 1) && keyValueCreators.get(0).isFullCopy;

        logger.info("Input path: {}", inputPath);
        logger.info("Output path: {}", outputPath);
        // read partition split keys
        List<RowKeyWritable> keys = new ArrayList<>();
        try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, partitionFilePath,
                sc.hadoopConfiguration())) {
            RowKeyWritable key = new RowKeyWritable();
            Writable value = NullWritable.get();
            while (reader.next(key, value)) {
                keys.add(key);
                logger.info(" ------- split key: {}", key);
                key = new RowKeyWritable(); // important, new an object!
            }
        }

        logger.info("There are {} split keys, totally {} hfiles", keys.size(), (keys.size() + 1));

        //HBase conf
        logger.info("Loading HBase configuration from:{}", hbaseConfFile);
        final Path hbaseConfFilePath = new Path(hbaseConfFile);
        final FileSystem hbaseClusterFs = hbaseConfFilePath.getFileSystem(sc.hadoopConfiguration());

        try (FSDataInputStream confInput = hbaseClusterFs.open(new Path(hbaseConfFile))) {
            Configuration hbaseJobConf = new Configuration();
            hbaseJobConf.addResource(confInput);
            hbaseJobConf.set("spark.hadoop.dfs.replication", "3"); // HFile, replication=3
            Job job = Job.getInstance(hbaseJobConf, cubeSegment.getStorageLocationIdentifier());

            FileOutputFormat.setOutputPath(job, new Path(outputPath));

            // inputPath has the same FileSystem as hbaseClusterFs when in HBase standalone mode
            JavaPairRDD<Text, Text> inputRDDs = SparkUtil.parseInputPath(inputPath, hbaseClusterFs, sc,
                    Text.class, Text.class);
            final JavaPairRDD<RowKeyWritable, KeyValue> hfilerdd;
            if (quickPath) {
                hfilerdd = inputRDDs
                        .mapToPair(new PairFunction<Tuple2<Text, Text>, RowKeyWritable, KeyValue>() {
                            @Override
                            public Tuple2<RowKeyWritable, KeyValue> call(Tuple2<Text, Text> textTextTuple2)
                                    throws Exception {
                                KeyValue outputValue = keyValueCreators.get(0).create(textTextTuple2._1,
                                        textTextTuple2._2.getBytes(), 0, textTextTuple2._2.getLength());
                                return new Tuple2<>(
                                        new RowKeyWritable(outputValue.createKeyOnly(false).getKey()),
                                        outputValue);
                            }
                        });
            } else {
                hfilerdd = inputRDDs
                        .flatMapToPair(new PairFlatMapFunction<Tuple2<Text, Text>, RowKeyWritable, KeyValue>() {
                            @Override
                            public Iterator<Tuple2<RowKeyWritable, KeyValue>> call(
                                    Tuple2<Text, Text> textTextTuple2) throws Exception {

                                List<Tuple2<RowKeyWritable, KeyValue>> result = Lists
                                        .newArrayListWithExpectedSize(cfNum);
                                Object[] inputMeasures = new Object[cubeDesc.getMeasures().size()];
                                inputCodec.decode(ByteBuffer.wrap(textTextTuple2._2.getBytes(), 0,
                                        textTextTuple2._2.getLength()), inputMeasures);

                                for (int i = 0; i < cfNum; i++) {
                                    KeyValue outputValue = keyValueCreators.get(i).create(textTextTuple2._1,
                                            inputMeasures);
                                    result.add(new Tuple2<>(
                                            new RowKeyWritable(outputValue.createKeyOnly(false).getKey()),
                                            outputValue));
                                }

                                return result.iterator();
                            }
                        });
            }

            hfilerdd.repartitionAndSortWithinPartitions(new HFilePartitioner(keys),
                    RowKeyWritable.RowKeyComparator.INSTANCE)
                    .mapToPair(
                            new PairFunction<Tuple2<RowKeyWritable, KeyValue>, ImmutableBytesWritable, KeyValue>() {
                                @Override
                                public Tuple2<ImmutableBytesWritable, KeyValue> call(
                                        Tuple2<RowKeyWritable, KeyValue> rowKeyWritableKeyValueTuple2)
                                        throws Exception {
                                    return new Tuple2<>(
                                            new ImmutableBytesWritable(
                                                    rowKeyWritableKeyValueTuple2._2.getKey()),
                                            rowKeyWritableKeyValueTuple2._2);
                                }
                            })
                    .saveAsNewAPIHadoopDataset(job.getConfiguration());
        }

        logger.info("HDFS: Number of bytes written={}", jobListener.metrics.getBytesWritten());

        Map<String, String> counterMap = Maps.newHashMap();
        counterMap.put(ExecutableConstants.HDFS_BYTES_WRITTEN,
                String.valueOf(jobListener.metrics.getBytesWritten()));

        // save counter to hdfs
        HadoopUtil.writeToSequenceFile(sc.hadoopConfiguration(), counterPath, counterMap);
    }
}

---

From source file:org.apache.lens.driver.hive.TestRemoteHiveDriver.java

License:Apache License

/**
 * Test hive driver persistence./*from www.  j  a  va  2 s. c  o m*/
 *
 * @throws Exception the exception
 */
@Test
public void testHiveDriverPersistence() throws Exception {
    System.out.println("@@@@ start_persistence_test");
    Configuration driverConf = new Configuration(remoteConf);
    driverConf.addResource("drivers/hive/hive1/hivedriver-site.xml");
    driverConf.setLong(HiveDriver.HS2_CONNECTION_EXPIRY_DELAY, 10000);
    driverConf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, false);

    final HiveDriver oldDriver = new HiveDriver();
    oldDriver.configure(driverConf, "hive", "hive1");

    queryConf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, false);
    queryConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
    QueryContext ctx = createContext("USE " + dataBase, queryConf, oldDriver);
    oldDriver.execute(ctx);
    Assert.assertEquals(0, oldDriver.getHiveHandleSize());

    String tableName = "test_hive_driver_persistence";

    // Create some ops with a driver
    String createTable = "CREATE TABLE IF NOT EXISTS " + tableName + "(ID STRING)";
    ctx = createContext(createTable, queryConf, oldDriver);
    oldDriver.execute(ctx);

    // Load some data into the table
    String dataLoad = "LOAD DATA LOCAL INPATH '" + TEST_DATA_FILE + "' OVERWRITE INTO TABLE " + tableName;
    ctx = createContext(dataLoad, queryConf, oldDriver);
    oldDriver.execute(ctx);

    queryConf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, true);
    queryConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true);
    // Fire two queries
    QueryContext ctx1 = createContext("SELECT * FROM " + tableName, queryConf, oldDriver);
    oldDriver.executeAsync(ctx1);
    QueryContext ctx2 = createContext("SELECT ID FROM " + tableName, queryConf, oldDriver);
    oldDriver.executeAsync(ctx2);
    Assert.assertEquals(2, oldDriver.getHiveHandleSize());

    byte[] ctx1bytes = persistContext(ctx1);
    byte[] ctx2bytes = persistContext(ctx2);

    // Write driver to stream
    ByteArrayOutputStream driverBytes = new ByteArrayOutputStream();
    ObjectOutputStream out = new ObjectOutputStream(driverBytes);
    try {
        oldDriver.writeExternal(out);
    } finally {
        out.close();
        driverBytes.close();
    }

    // Create another driver from the stream
    ByteArrayInputStream driverInput = new ByteArrayInputStream(driverBytes.toByteArray());
    HiveDriver newDriver = new HiveDriver();
    newDriver.readExternal(new ObjectInputStream(driverInput));
    newDriver.configure(driverConf, "hive", "hive1");
    driverInput.close();

    ctx1 = readContext(ctx1bytes, newDriver);
    ctx2 = readContext(ctx2bytes, newDriver);

    Assert.assertEquals(2, newDriver.getHiveHandleSize());

    validateExecuteAsync(ctx1, DriverQueryState.SUCCESSFUL, true, false, newDriver);
    validateExecuteAsync(ctx2, DriverQueryState.SUCCESSFUL, true, false, newDriver);
}

---

From source file:org.apache.lens.server.session.LensSessionImpl.java

License:Apache License

/**
 * Creates the default conf./*from   www. j a v  a  2s  .  c  o m*/
 *
 * @return the configuration
 */
public static synchronized Configuration createDefaultConf() {
    if (sessionDefaultConfig == null) {
        Configuration conf = new Configuration(false);
        conf.addResource("lenssession-default.xml");
        conf.addResource("lens-site.xml");
        sessionDefaultConfig = new Configuration(false);
        for (Map.Entry<String, String> prop : conf) {
            if (!prop.getKey().startsWith(LensConfConstants.SERVER_PFX)) {
                sessionDefaultConfig.set(prop.getKey(), prop.getValue());
            }
        }
    }
    //Not exposing sessionDefaultConfig directly to insulate it form modifications
    return new Configuration(sessionDefaultConfig);
}

---

From source file:org.apache.nifi.hbase.HBase_1_1_2_ClientService.java

License:Apache License

protected Configuration getConfigurationFromFiles(final String configFiles) {
    final Configuration hbaseConfig = HBaseConfiguration.create();
    if (StringUtils.isNotBlank(configFiles)) {
        for (final String configFile : configFiles.split(",")) {
            hbaseConfig.addResource(new Path(configFile.trim()));
        }//from   w  ww  .  j ava 2s. c  o  m
    }
    return hbaseConfig;
}

---

From source file:org.apache.nifi.processors.hadoop.AbstractHadoopProcessor.java

License:Apache License

private static Configuration getConfigurationFromResources(final Configuration config, String configResources)
        throws IOException {
    boolean foundResources = false;
    if (null != configResources) {
        String[] resources = configResources.split(",");
        for (String resource : resources) {
            config.addResource(new Path(resource.trim()));
            foundResources = true;//from  w w w .  ja  va  2 s.com
        }
    }

    if (!foundResources) {
        // check that at least 1 non-default resource is available on the classpath
        String configStr = config.toString();
        for (String resource : configStr.substring(configStr.indexOf(":") + 1).split(",")) {
            if (!resource.contains("default") && config.getResource(resource.trim()) != null) {
                foundResources = true;
                break;
            }
        }
    }

    if (!foundResources) {
        throw new IOException(
                "Could not find any of the " + HADOOP_CONFIGURATION_RESOURCES.getName() + " on the classpath");
    }
    return config;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »