Example usage for org.apache.hadoop.conf Configuration addResource

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration addResource.

Prototype

public void addResource(Configuration conf) 

Source Link

Document

Add a configuration resource.

Usage

From source file:com.example.mydtapp.JdbcInputAppTest.java

License:Apache License

@Test
public void testApplication() throws Exception {
    try {/*from  ww  w .jav  a 2s  . co m*/
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
        lma.prepareDAG(new JdbcHDFSApp(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.runAsync();

        // wait for output files to roll      
        Thread.sleep(5000);

        String[] extensions = { "dat.0", "tmp" };
        Collection<File> list = FileUtils.listFiles(new File(FILE_NAME), extensions, false);
        Assert.assertEquals("Records in file", 10, FileUtils.readLines(list.iterator().next()).size());

    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

---

From source file:com.example.mydtapp.JdbcOperatorTest.java

License:Apache License

@Test
public void testApplication() throws Exception {
    try {//from  w w  w . j a  v  a 2s  .c  o m
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
        lma.prepareDAG(new JdbcToJdbcApp(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.runAsync();

        // wait for records to be added to table    
        Thread.sleep(5000);

        Assert.assertEquals("Events in store", 10, getNumOfEventsInStore());
        cleanTable();

    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

---

From source file:com.example.neuralnet.ApplicationTest.java

@Test
public void testApplication() throws IOException, Exception {
    try {/*from   w  ww. j  av a  2  s  .  co  m*/
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
        lma.prepareDAG(new Application(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.run(1000000); // runs for 10 seconds and quits
    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

---

From source file:com.facebook.presto.example.ExampleClient.java

License:Apache License

private static Map<String, Map<String, ExampleTable>> lookupSchemas(URI metadataUri,
        JsonCodec<Map<String, List<ExampleTable>>> catalogCodec) throws IOException {
    String json = null;/*from w  w  w  .j  av  a 2s  .c o m*/
    if (metadataUri.getScheme().equalsIgnoreCase("hdfs")) {
        // schema file on hdfs
        String hdfsSiteLocation = "/etc/hadoop/conf/hdfs-site.xml";
        String coreSiteLocation = "/etc/hadoop/conf/core-site.xml";

        Configuration conf = new Configuration();
        final Path hdfsConf = new Path(hdfsSiteLocation);
        final Path coreConf = new Path(coreSiteLocation);
        conf.addResource(hdfsConf);
        conf.addResource(coreConf);

        Path schemaPath = new Path(metadataUri);

        FileSystem fs = FileSystem.get(conf);

        if (!fs.exists(schemaPath)) {
            byte[] schemaBytes = ByteStreams.toByteArray(fs.open(schemaPath));
            json = new String(schemaBytes, UTF_8);
        }
    } else {
        URL result = metadataUri.toURL();
        json = Resources.toString(result, UTF_8);
    }
    Map<String, List<ExampleTable>> catalog = catalogCodec.fromJson(json);

    return ImmutableMap.copyOf(transformValues(catalog, resolveAndIndexTables(metadataUri)));
}

---

From source file:com.facebook.presto.hive.HdfsConfiguration.java

License:Apache License

protected Configuration createConfiguration() {
    Configuration config = new Configuration();

    if (resourcePaths != null) {
        for (String resourcePath : resourcePaths) {
            config.addResource(new Path(resourcePath));
        }/* ww w. ja va 2s . c o  m*/
    }

    // this is to prevent dfs client from doing reverse DNS lookups to determine whether nodes are rack local
    config.setClass("topology.node.switch.mapping.impl", NoOpDNSToSwitchMapping.class,
            DNSToSwitchMapping.class);

    if (socksProxy != null) {
        config.setClass("hadoop.rpc.socket.factory.class.default", SocksSocketFactory.class,
                SocketFactory.class);
        config.set("hadoop.socks.server", socksProxy.toString());
    }

    if (domainSocketPath != null) {
        config.setStrings("dfs.domain.socket.path", domainSocketPath);
    }

    // only enable short circuit reads if domain socket path is properly configured
    if (!config.get("dfs.domain.socket.path", "").trim().isEmpty()) {
        config.setBooleanIfUnset("dfs.client.read.shortcircuit", true);
    }

    config.setInt("dfs.socket.timeout", Ints.checkedCast(dfsTimeout.toMillis()));
    config.setInt("ipc.ping.interval", Ints.checkedCast(dfsTimeout.toMillis()));
    config.setInt("ipc.client.connect.timeout", Ints.checkedCast(dfsConnectTimeout.toMillis()));
    config.setInt("ipc.client.connect.max.retries", dfsConnectMaxRetries);

    // re-map filesystem schemes to match Amazon Elastic MapReduce
    config.set("fs.s3.impl", PrestoS3FileSystem.class.getName());
    config.set("fs.s3n.impl", PrestoS3FileSystem.class.getName());
    config.set("fs.s3bfs.impl", "org.apache.hadoop.fs.s3.S3FileSystem");

    // set AWS credentials for S3
    for (String scheme : ImmutableList.of("s3", "s3bfs", "s3n")) {
        if (s3AwsAccessKey != null) {
            config.set(format("fs.%s.awsAccessKeyId", scheme), s3AwsAccessKey);
        }
        if (s3AwsSecretKey != null) {
            config.set(format("fs.%s.awsSecretAccessKey", scheme), s3AwsSecretKey);
        }
    }

    // set config for S3
    config.setBoolean(PrestoS3FileSystem.S3_SSL_ENABLED, s3SslEnabled);
    config.setInt(PrestoS3FileSystem.S3_MAX_CLIENT_RETRIES, s3MaxClientRetries);
    config.setInt(PrestoS3FileSystem.S3_MAX_ERROR_RETRIES, s3MaxErrorRetries);
    config.set(PrestoS3FileSystem.S3_CONNECT_TIMEOUT, s3ConnectTimeout.toString());
    config.set(PrestoS3FileSystem.S3_STAGING_DIRECTORY, s3StagingDirectory.toString());

    updateConfiguration(config);

    return config;
}

---

From source file:com.facebook.presto.hive.HdfsConfigurationUpdater.java

License:Apache License

public void updateConfiguration(Configuration config) {
    if (resourcePaths != null) {
        for (String resourcePath : resourcePaths) {
            config.addResource(new Path(resourcePath));
        }/*w w  w . j  a v  a  2 s  . c  o  m*/
    }

    // this is to prevent dfs client from doing reverse DNS lookups to determine whether nodes are rack local
    config.setClass("topology.node.switch.mapping.impl", NoOpDNSToSwitchMapping.class,
            DNSToSwitchMapping.class);

    if (socksProxy != null) {
        config.setClass("hadoop.rpc.socket.factory.class.default", SocksSocketFactory.class,
                SocketFactory.class);
        config.set("hadoop.socks.server", socksProxy.toString());
    }

    if (domainSocketPath != null) {
        config.setStrings("dfs.domain.socket.path", domainSocketPath);
    }

    // only enable short circuit reads if domain socket path is properly configured
    if (!config.get("dfs.domain.socket.path", "").trim().isEmpty()) {
        config.setBooleanIfUnset("dfs.client.read.shortcircuit", true);
    }

    config.setInt("dfs.socket.timeout", toIntExact(dfsTimeout.toMillis()));
    config.setInt("ipc.ping.interval", toIntExact(ipcPingInterval.toMillis()));
    config.setInt("ipc.client.connect.timeout", toIntExact(dfsConnectTimeout.toMillis()));
    config.setInt("ipc.client.connect.max.retries", dfsConnectMaxRetries);

    // re-map filesystem schemes to match Amazon Elastic MapReduce
    config.set("fs.s3.impl", PrestoS3FileSystem.class.getName());
    config.set("fs.s3a.impl", PrestoS3FileSystem.class.getName());
    config.set("fs.s3n.impl", PrestoS3FileSystem.class.getName());
    config.set("fs.s3bfs.impl", "org.apache.hadoop.fs.s3.S3FileSystem");

    // set AWS credentials for S3
    if (s3AwsAccessKey != null) {
        config.set(PrestoS3FileSystem.S3_ACCESS_KEY, s3AwsAccessKey);
        config.set("fs.s3bfs.awsAccessKeyId", s3AwsAccessKey);
    }
    if (s3AwsSecretKey != null) {
        config.set(PrestoS3FileSystem.S3_SECRET_KEY, s3AwsSecretKey);
        config.set("fs.s3bfs.awsSecretAccessKey", s3AwsSecretKey);
    }
    if (s3Endpoint != null) {
        config.set(PrestoS3FileSystem.S3_ENDPOINT, s3Endpoint);
        config.set("fs.s3bfs.Endpoint", s3Endpoint);
    }
    if (s3SignerType != null) {
        config.set(PrestoS3FileSystem.S3_SIGNER_TYPE, s3SignerType.getSignerType());
    }

    config.setInt("fs.cache.max-size", fileSystemMaxCacheSize);

    configureCompression(config, compressionCodec);

    // set config for S3
    config.setBoolean(PrestoS3FileSystem.S3_USE_INSTANCE_CREDENTIALS, s3UseInstanceCredentials);
    config.setBoolean(PrestoS3FileSystem.S3_SSL_ENABLED, s3SslEnabled);
    config.setBoolean(PrestoS3FileSystem.S3_SSE_ENABLED, s3SseEnabled);
    if (s3EncryptionMaterialsProvider != null) {
        config.set(PrestoS3FileSystem.S3_ENCRYPTION_MATERIALS_PROVIDER, s3EncryptionMaterialsProvider);
    }
    if (s3KmsKeyId != null) {
        config.set(PrestoS3FileSystem.S3_KMS_KEY_ID, s3KmsKeyId);
    }
    config.setInt(PrestoS3FileSystem.S3_MAX_CLIENT_RETRIES, s3MaxClientRetries);
    config.setInt(PrestoS3FileSystem.S3_MAX_ERROR_RETRIES, s3MaxErrorRetries);
    config.set(PrestoS3FileSystem.S3_MAX_BACKOFF_TIME, s3MaxBackoffTime.toString());
    config.set(PrestoS3FileSystem.S3_MAX_RETRY_TIME, s3MaxRetryTime.toString());
    config.set(PrestoS3FileSystem.S3_CONNECT_TIMEOUT, s3ConnectTimeout.toString());
    config.set(PrestoS3FileSystem.S3_SOCKET_TIMEOUT, s3SocketTimeout.toString());
    config.set(PrestoS3FileSystem.S3_STAGING_DIRECTORY, s3StagingDirectory.toString());
    config.setInt(PrestoS3FileSystem.S3_MAX_CONNECTIONS, s3MaxConnections);
    config.setLong(PrestoS3FileSystem.S3_MULTIPART_MIN_FILE_SIZE, s3MultipartMinFileSize.toBytes());
    config.setLong(PrestoS3FileSystem.S3_MULTIPART_MIN_PART_SIZE, s3MultipartMinPartSize.toBytes());
    config.setBoolean(PrestoS3FileSystem.S3_PIN_CLIENT_TO_CURRENT_REGION, pinS3ClientToCurrentRegion);
    config.set(PrestoS3FileSystem.S3_USER_AGENT_PREFIX, s3UserAgentPrefix);
}

---

From source file:com.firewallid.util.FIConfiguration.java

private static Configuration addFirewallIndonesiaResources(Configuration conf) {
    conf.addResource("firewallid-default.xml");
    return conf;/*  www  . j a v a2  s. c  o  m*/
}

---

From source file:com.fline.hadoop.data.client.DataTransporter.java

License:Apache License

/**
 * trans rdb data to hbase./*from w w w . j  av  a 2  s .  c om*/
 * 
 * @param connectionurl
 *            rdb url
 * @param driver
 *            rdb driver class. such as "com.mysql.jdbc.Driver"
 * @param username
 *            rdb login username
 * @param password
 *            rdb login password
 * @param tablename
 *            rdb table.
 * @param rdbcolumns
 *            rdb table column which selected to write to hbase
 * @param partitioncolumn
 *            the column which can be used to select data by set a start
 *            value and end value
 * @param linenum
 *            record num
 * @param hbasetable
 *            output hbase table name
 * @param hbasecolumns
 *            the columns corresponding to rdb columns
 * @param rowkeyparam
 *            hbase rowkey generate param.
 * @param solrmasterurl
 *            such as http://fdp-master:8983/solr/
 * @param label
 *            solr label used for search
 * @param listener
 * @throws Exception
 */
public static void transRDB2HBASEWithIndexOnSolr(String connectionurl, String driver, String username,
        String password, String tablename, String rdbcolumns, String partitioncolumn, int linenum,
        String hbasetable, String hbasecolumns, String rowkeyparam, String solrmasterurl, String label,
        DataProgressListener listener) throws Exception {
    HashMap<String, String> rdbconfigMap = new HashMap<String, String>();
    rdbconfigMap.put(RDBDataSource.CONFIG_JDBC_CONNECTIONSTRING, connectionurl);
    rdbconfigMap.put(RDBDataSource.CONFIG_JDBC_DRIVER, driver);
    rdbconfigMap.put(RDBDataSource.CONFIG_JDBC_USERNAME, username);
    rdbconfigMap.put(RDBDataSource.CONFIG_JDBC_USERPASSWD, password);
    // configMap.put(RDBInputDataSource.CONFIG_JDBC_SCHEMA, "");
    rdbconfigMap.put(RDBInputDataSource.CONFIG_JDBC_COLUMNS, rdbcolumns);
    //       rdbconfigMap.put(CONFIG_MAPNUM, "2000");
    // rdbconfigMap.put("fromJobConfig.sql", "1=1 limit " + linenum);
    rdbconfigMap.put(CONFIG_MAPNUM, String.valueOf(linenum / 1000 + 1));
    System.out.println("config_mapnum*********************:" + linenum);
    if (linenum <= 0) {
        rdbconfigMap.put(RDBInputDataSource.CONFIG_JDBC_TABLE, tablename);
    } else {
        rdbconfigMap.put(RDBInputDataSource.CONFIG_JDBC_TABLE,
                "(select * from " + tablename + " limit " + linenum + " ) as temptable");
    }
    rdbconfigMap.put(RDBInputDataSource.CONFIG_JDBC_PARTITIONCOLUMN, partitioncolumn);
    InputDataSource rdb = InputDataSourceFactory.createInputDataSource(Constant.RDBS_DATASOURCE, rdbconfigMap);
    // HBASE CONFIG
    HashMap<String, String> hbaseconfigMap = new HashMap<String, String>();
    Configuration conf = new Configuration();
    conf.addResource(new FileInputStream(
            DataTransporter.class.getClassLoader().getResource("").getPath() + "hbase-site.xml"));
    hbaseconfigMap.put(HBaseDataSource.CONFIG_HBASE_ZOOKEEPERLIST, conf.get("hbase.zookeeper.quorum"));
    hbaseconfigMap.put(HBaseDataSource.CONFIG_HBASE_ZKNODE, conf.get("zookeeper.znode.parent"));
    hbaseconfigMap.put(HBaseOutputDataSource.CONFIG_HBASE_TABLENAME, hbasetable);
    hbaseconfigMap.put(HBaseOutputDataSource.CONFIG_HBASE_COLUMNSMAP, hbasecolumns);
    hbaseconfigMap.put(HBaseOutputDataSource.CONFIG_HBASE_ROWKEYGENEREATEDWAY,
            HBaseOutputDataSource.ROWKEY_GENERATED_BY_NORMAL);
    hbaseconfigMap.put(HBaseOutputDataSource.CONFIG_HBASE_ROWKEYPARAMS, rowkeyparam);
    OutputDataSource hbase = OutputDataSourceFactory.createOutputDataSource(Constant.HBASE_DATASOURCE,
            hbaseconfigMap);
    // solr meta store
    HttpSolrClient metaclient = new HttpSolrClient(solrmasterurl + "core_for_Meta");
    List<SolrInputDocument> sidocs = new ArrayList<SolrInputDocument>();
    SolrInputDocument sidoc = new SolrInputDocument();
    sidoc.addField("rdbtablename", tablename);
    sidoc.addField("rdbtablecols", generateDynamicSolrColumns(rdbcolumns));
    sidocs.add(sidoc);
    metaclient.add(sidocs);
    metaclient.commit();
    metaclient.close();
    // solr config
    String rdbname = null;
    if (driver.contains("mysql")) {
        rdbname = connectionurl.substring(connectionurl.lastIndexOf('/') + 1);
    } else if (driver.contains("oracle")) {
        rdbname = connectionurl.substring(connectionurl.lastIndexOf(':') + 1);
    }
    HashMap<String, String> solrconfig = new HashMap<String, String>();
    solrconfig.put(SolrDataSource.CONFIG_SOLR_MASTERURL, solrmasterurl);
    solrconfig.put(SolrOutputDataSource.CONFIG_SOLR_COLUMNS,
            "label=" + label + ",linecount=" + linenum + ",rdbname=" + rdbname + ",rdbtablename=" + hbasetable
                    + ",createdTime=" + System.currentTimeMillis() + ",sourceType=2@di_v@"
                    + generateDynamicSolrColumns(rdbcolumns));
    System.out.println(SolrOutputDataSource.CONFIG_SOLR_COLUMNS + "\t" + "label=" + label + ",linecount="
            + linenum + ",rdbname=" + rdbname + ",rdbtablename=" + tablename + ",createdTime="
            + System.currentTimeMillis() + ",sourceType=2@di_v@" + generateDynamicSolrColumns(rdbcolumns));
    solrconfig.put(SolrOutputDataSource.CONFIG_SOLR_INSTANCE, "core_for_searchDB");
    OutputDataSource solr = new SolrOutputDataSource(solrconfig, Constant.SOLR_DATASOURCE_NAME);
    DataTransporter.createTransJob(rdb, hbase, null);
    DataTransporter.createTransJob(rdb, solr, listener);
}

---

From source file:com.fline.hadoop.data.client.DataTransporter.java

License:Apache License

/**
 * trans rdb data to hbase with incre condition .
 * //w  w w  . j a  v a  2s.c  o  m
 * @param connectionurl
 *            rdb url
 * @param driver
 *            rdb driver class. such as "com.mysql.jdbc.Driver"
 * @param username
 *            rdb login username
 * @param password
 *            rdb login password
 * @param tablename
 *            rdb table.
 * @param rdbcolumns
 *            rdb table column which selected to write to hbase
 * @param partitioncolumn
 *            the column which can be used to select data by set a start
 *            value and end value
 * @param increCheckColumn
 *            incre check column.
 * @param increLastValue
 *            start column value.
 * @param linenum
 *            record num
 * @param hbasetable
 *            output hbase table name
 * @param hbasecolumns
 *            the columns corresponding to rdb columns
 * @param rowkeyparam
 *            hbase rowkey generate param.
 * @param solrmasterurl
 *            such as http://fdp-master:8983/solr/
 * @param label
 *            solr label used for search
 * @param listener
 * @throws Exception
 */
public static void transRDBIncre2HBASEWithIndexOnSolr(String connectionurl, String driver, String username,
        String password, String tablename, String rdbcolumns, String partitioncolumn, String increCheckColumn,
        String increLastValue, int linenum, String hbasetable, String hbasecolumns, String rowkeyparam,
        String solrmasterurl, String label, DataProgressListener listener) throws Exception {
    HashMap<String, String> rdbconfigMap = new HashMap<String, String>();
    rdbconfigMap.put(RDBDataSource.CONFIG_JDBC_CONNECTIONSTRING, connectionurl);
    rdbconfigMap.put(RDBDataSource.CONFIG_JDBC_DRIVER, driver);
    rdbconfigMap.put(RDBDataSource.CONFIG_JDBC_USERNAME, username);
    rdbconfigMap.put(RDBDataSource.CONFIG_JDBC_USERPASSWD, password);
    // configMap.put(RDBInputDataSource.CONFIG_JDBC_SCHEMA, "");
    rdbconfigMap.put(RDBInputDataSource.CONFIG_JDBC_COLUMNS, rdbcolumns);
    rdbconfigMap.put("fromJobConfig.boundaryQuery", "select min(" + partitioncolumn + "),max(" + partitioncolumn
            + ") from " + tablename + " where " + increCheckColumn + " >= " + increLastValue);
    rdbconfigMap.put(CONFIG_MAPNUM, String.valueOf(linenum / 1000 + 1));
    System.out.println("config_mapnum*********************:" + linenum);
    if (linenum <= 0) {
        rdbconfigMap.put(RDBInputDataSource.CONFIG_JDBC_TABLE, tablename);
    } else {
        rdbconfigMap.put(RDBInputDataSource.CONFIG_JDBC_TABLE,
                "(select * from " + tablename + " limit " + linenum + " ) as temptable");
    }
    rdbconfigMap.put(RDBInputDataSource.CONFIG_JDBC_PARTITIONCOLUMN, partitioncolumn);
    InputDataSource rdb = InputDataSourceFactory.createInputDataSource(Constant.RDBS_DATASOURCE, rdbconfigMap);
    // HBASE CONFIG
    HashMap<String, String> hbaseconfigMap = new HashMap<String, String>();
    Configuration conf = new Configuration();
    conf.addResource(new FileInputStream(
            DataTransporter.class.getClassLoader().getResource("").getPath() + "hbase-site.xml"));
    hbaseconfigMap.put(HBaseDataSource.CONFIG_HBASE_ZOOKEEPERLIST, conf.get("hbase.zookeeper.quorum"));
    hbaseconfigMap.put(HBaseDataSource.CONFIG_HBASE_ZKNODE, conf.get("zookeeper.znode.parent"));
    hbaseconfigMap.put(HBaseOutputDataSource.CONFIG_HBASE_TABLENAME, hbasetable);
    hbaseconfigMap.put(HBaseOutputDataSource.CONFIG_HBASE_COLUMNSMAP, hbasecolumns);
    hbaseconfigMap.put(HBaseOutputDataSource.CONFIG_HBASE_ROWKEYGENEREATEDWAY,
            HBaseOutputDataSource.ROWKEY_GENERATED_BY_NORMAL);
    hbaseconfigMap.put(HBaseOutputDataSource.CONFIG_HBASE_ROWKEYPARAMS, rowkeyparam);
    OutputDataSource hbase = OutputDataSourceFactory.createOutputDataSource(Constant.HBASE_DATASOURCE,
            hbaseconfigMap);
    // solr meta store
    HttpSolrClient metaclient = new HttpSolrClient(solrmasterurl + "core_for_Meta");
    List<SolrInputDocument> sidocs = new ArrayList<SolrInputDocument>();
    SolrInputDocument sidoc = new SolrInputDocument();
    sidoc.addField("rdbtablename", tablename);
    sidoc.addField("rdbtablecols", generateDynamicSolrColumns(rdbcolumns));
    sidocs.add(sidoc);
    metaclient.add(sidocs);
    metaclient.commit();
    metaclient.close();
    // solr config
    HashMap<String, String> solrconfig = new HashMap<String, String>();
    solrconfig.put(SolrDataSource.CONFIG_SOLR_MASTERURL, solrmasterurl);
    solrconfig.put(SolrOutputDataSource.CONFIG_SOLR_COLUMNS,
            "label=" + label + ",linecount=" + linenum + ",rdbname="
                    + connectionurl.substring(connectionurl.lastIndexOf('/')) + ",rdbtablename=" + tablename
                    + ",createdTime=" + System.currentTimeMillis() + ",sourceType=2@di_v@"
                    + generateDynamicSolrColumns(rdbcolumns));
    solrconfig.put(SolrOutputDataSource.CONFIG_SOLR_INSTANCE, "core_for_searchDB");
    OutputDataSource solr = new SolrOutputDataSource(solrconfig, Constant.SOLR_DATASOURCE_NAME);
    DataTransporter.createTransJob(rdb, hbase, listener);
    DataTransporter.createTransJob(rdb, solr, null);
}

---

From source file:com.fline.hadoop.data.client.DataTransporter.java

License:Apache License

public static void uploadFile2HBASE(String localfilepath, String filetype, String hbasetable,
        String hbasecolumns, String rowkeyparam, DataProgressListener listener) throws Exception {
    HashMap<String, String> configMap = new HashMap<String, String>();
    configMap.put(FileDataSource.CONFIG_FILE_PATH, localfilepath);
    if (filetype.equals("csv")) {
        configMap.put(FileInputDataSource.CONFIG_ANALYSIS_READER, FileInputDataSource.LINE_READER);
        configMap.put(FileInputDataSource.CONFIG_ANALYSZER_DRIVER, FileInputDataSource.CSV_ANALYZER);
    } else if (filetype.equals("log")) {
        configMap.put(FileInputDataSource.CONFIG_ANALYSIS_READER, FileInputDataSource.LINE_READER);
        configMap.put(FileInputDataSource.CONFIG_ANALYSZER_DRIVER, FileInputDataSource.LOG_ANALYZER);
    } else {/*from   w ww  .ja v a  2s .c om*/
        throw new Exception("Unsupported filetype = " + filetype + "\t to upload file to hbase.");
    }
    InputDataSource inputdatasource = InputDataSourceFactory
            .createInputDataSource(Constant.NORMAL_FILE_DATASOURCE, configMap);

    // HBASE CONFIG
    HashMap<String, String> hbaseconfigMap = new HashMap<String, String>();
    Configuration conf = new Configuration();
    conf.addResource(new FileInputStream(
            DataTransporter.class.getClassLoader().getResource("").getPath() + "hbase-site.xml"));
    hbaseconfigMap.put(HBaseDataSource.CONFIG_HBASE_ZOOKEEPERLIST, conf.get("hbase.zookeeper.quorum"));
    hbaseconfigMap.put(HBaseDataSource.CONFIG_HBASE_ZKNODE, conf.get("zookeeper.znode.parent"));
    hbaseconfigMap.put(HBaseOutputDataSource.CONFIG_HBASE_TABLENAME, hbasetable);
    hbaseconfigMap.put(HBaseOutputDataSource.CONFIG_HBASE_COLUMNSMAP, hbasecolumns);
    hbaseconfigMap.put(HBaseOutputDataSource.CONFIG_HBASE_ROWKEYGENEREATEDWAY,
            HBaseOutputDataSource.ROWKEY_GENERATED_BY_NORMAL);
    hbaseconfigMap.put(HBaseOutputDataSource.CONFIG_HBASE_ROWKEYPARAMS, rowkeyparam);
    OutputDataSource hbase = OutputDataSourceFactory.createOutputDataSource(Constant.HBASE_DATASOURCE,
            hbaseconfigMap);
    DataTransporter.createTransJob(inputdatasource, hbase, listener);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »