List of usage examples for org.apache.hadoop.conf Configuration setClass
public void setClass(String name, Class<?> theClass, Class<?> xface)
name property to the name of a theClass implementing the given interface xface. From source file:com.splicemachine.derby.stream.spark.SparkInsertTableWriterBuilder.java
License:Apache License
@SuppressWarnings("unchecked") @Override//from www . ja v a2 s. c o m public DataSetWriter build() throws StandardException { if (operationContext.getOperation() != null) { operationContext.getOperation().fireBeforeStatementTriggers(); } final Configuration conf = new Configuration(HConfiguration.unwrapDelegate()); try { TableWriterUtils.serializeInsertTableWriterBuilder(conf, this); } catch (IOException e) { throw Exceptions.parseException(e); } conf.setClass(JobContext.OUTPUT_FORMAT_CLASS_ATTR, SMOutputFormat.class, SMOutputFormat.class); return new InsertDataSetWriter<>(rdd, operationContext, conf, pkCols, tableVersion, execRowDefinition, autoIncrementRowLocationArray, spliceSequences, heapConglom, isUpsert); }
From source file:com.splicemachine.derby.stream.spark.SparkUpdateTableWriterBuilder.java
License:Apache License
@Override public DataSetWriter build() throws StandardException { try {/* w w w . j a v a 2 s . c om*/ if (operationContext.getOperation() != null) { operationContext.getOperation().fireBeforeStatementTriggers(); } Configuration conf = new Configuration(HConfiguration.unwrapDelegate()); TableWriterUtils.serializeUpdateTableWriterBuilder(conf, this); conf.setClass(JobContext.OUTPUT_FORMAT_CLASS_ATTR, SMOutputFormat.class, SMOutputFormat.class); return new SparkUpdateDataSetWriter<>(rdd, operationContext, conf, heapConglom, formatIds, columnOrdering, pkCols, pkColumns, tableVersion, execRowDefinition, heapList); } catch (Exception e) { throw new RuntimeException(e); } }
From source file:com.splicemachine.test.SpliceTestPlatformConfig.java
License:Apache License
public static Configuration create(String hbaseRootDirUri, Integer masterPort, Integer masterInfoPort, Integer regionServerPort, Integer regionServerInfoPort, Integer derbyPort, boolean failTasksRandomly) { Configuration config = HConfiguration.unwrapDelegate(); config.set(SQLConfiguration.STORAGE_FACTORY_HOME, hbaseRootDirUri); ////from ww w . j a v a 2 s .co m // Coprocessors // config.set("hbase.coprocessor.regionserver.classes", getRegionServerCoprocessorsAsString()); config.set("hbase.coprocessor.region.classes", getRegionCoprocessorsAsString()); config.set("hbase.coprocessor.master.classes", getMasterCoprocessorsAsString()); // // Networking // config.set("hbase.zookeeper.quorum", "127.0.0.1:2181"); config.setInt("hbase.master.port", masterPort); config.setInt("hbase.master.info.port", masterInfoPort); config.setInt("hbase.regionserver.port", regionServerPort); config.setInt("hbase.regionserver.info.port", regionServerInfoPort); config.setInt("hbase.master.jmx.port", HConfiguration.DEFAULT_JMX_BIND_PORT); // this is set because the HBase master and regionserver are running on the same machine and in the same JVM config.setInt(SQLConfiguration.NETWORK_BIND_PORT, derbyPort); config.setClass(DefaultStoreEngine.DEFAULT_COMPACTOR_CLASS_KEY, SpliceDefaultCompactor.class, Compactor.class); // config.setClass(ConsistencyControlUtils.MVCC_IMPL, SIMultiVersionConsistencyControl.class, ConsistencyControl.class); config.setClass(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, SpliceDefaultCompactionPolicy.class, CompactionPolicy.class); // // Networking -- interfaces // // force use of loop back interface on MacOSX, else don't set it // if (System.getProperty("os.name").contains("Mac") ) { // String interfaceName = "lo0"; // config.set("hbase.zookeeper.dns.interface", interfaceName); // config.set("hbase.master.dns.interface", interfaceName); // config.set("hbase.regionserver.dns.interface", interfaceName); // } // // File System // config.set("fs.defaultFS", "file:///"); // MapR Hack, tells it local filesystem // fs.default.name is deprecated config.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); config.setDouble("yarn.nodemanager.resource.io-spindles", 2.0); config.set("fs.default.name", "file:///"); config.set("yarn.nodemanager.container-executor.class", "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor"); // Must allow Cygwin instance to config its own rootURI if (!"CYGWIN".equals(hbaseRootDirUri)) { config.set("hbase.rootdir", hbaseRootDirUri); } // // Threads, timeouts // config.setLong("hbase.rpc.timeout", MINUTES.toMillis(2)); config.setLong("hbase.client.scanner.timeout.period", MINUTES.toMillis(2)); // hbase.regionserver.lease.period is deprecated config.setLong("hbase.client.operation.timeout", MINUTES.toMillis(2)); config.setLong("hbase.regionserver.handler.count", 200); config.setLong("hbase.regionserver.msginterval", 1000); config.setLong("hbase.master.event.waiting.time", 20); config.setLong("hbase.master.lease.thread.wakefrequency", SECONDS.toMillis(3)); // config.setBoolean("hbase.master.loadbalance.bytable",true); config.setInt("hbase.balancer.period", 5000); config.setLong("hbase.server.thread.wakefrequency", SECONDS.toMillis(1)); config.setLong("hbase.client.pause", 100); // // Compaction Controls // config.setLong("hbase.hstore.compaction.min", 5); // min number of eligible files before we compact config.setLong("hbase.hstore.compaction.max", 10); // max files to be selected for a single minor compaction config.setLong("hbase.hstore.compaction.min.size", 16 * MiB); // store files smaller than this will always be eligible for minor compaction. HFiles this size or larger are evaluated by hbase.hstore.compaction.ratio to determine if they are eligible config.setLong("hbase.hstore.compaction.max.size", 248 * MiB); // store files larger than this will be excluded from compaction config.setFloat("hbase.hstore.compaction.ratio", 1.25f); // default is 1.2f, at one point we had this set to 0.25f and 25f (which was likely a typo) // // Memstore, store files, splits // config.setLong(HConstants.HREGION_MAX_FILESIZE, 32 * MiB); // hbase.hregion.max.filesize config.setLong("hbase.hregion.memstore.flush.size", 128 * MiB); // was 512 MiB config.setLong("hbase.hregion.memstore.block.multiplier", 4); config.setFloat("hbase.regionserver.global.memstore.size", 0.25f); // set mem store to 25% of heap config.setLong("hbase.hstore.blockingStoreFiles", 20); // config.set("hbase.regionserver.region.split.policy", "org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy"); // change default split policy. this makes more sense for a standalone/single regionserver // Support SI //config.setClass(HConstants.MVCC_IMPL, SIMultiVersionConsistencyControl.class, ConsistencyControl.class); // // HFile // config.setInt("hfile.index.block.max.size", 16 * 1024); // 16KiB config.setFloat("hfile.block.cache.size", 0.25f); // set block cache to 25% of heap config.setFloat("io.hfile.bloom.error.rate", (float) 0.005); config.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, true); // hfile.block.bloom.cacheonwrite //config.set("hbase.master.hfilecleaner.plugins", getHFileCleanerAsString()); config.set("hbase.master.hfilecleaner.plugins", getHFileCleanerAsString()); // // Misc // config.set("hbase.cluster.distributed", "true"); // don't start zookeeper for us config.set("hbase.master.distributed.log.splitting", "false"); // TODO: explain why we are setting this // AWS Credentials for test... // config.set(ACCESS_KEY, "AKIAJ6HBMCK5ALHVBFPQ"); config.set(SECRET_KEY, "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"); // // Splice // config.setLong("splice.ddl.drainingWait.maximum", SECONDS.toMillis(15)); // wait 15 seconds before bailing on bad ddl statements config.setLong("splice.ddl.maxWaitSeconds", 120000); // // Snapshots // config.setBoolean("hbase.snapshot.enabled", true); HConfiguration.reloadConfiguration(config); return HConfiguration.unwrapDelegate(); }
From source file:com.streamsets.pipeline.lib.hdfs.common.HdfsBaseConfigBean.java
License:Apache License
protected Configuration getHadoopConfiguration(Stage.Context context, List<Stage.ConfigIssue> issues) { Configuration conf = new Configuration(); conf.setClass("fs.file.impl", RawLocalFileSystem.class, FileSystem.class); //We handle the file system close ourselves in destroy //If enabled, Also this will cause issues (not allow us to rename the files on destroy call) // when we run a shutdown hook on app kill //See https://issues.streamsets.com/browse/SDC-4057 conf.setBoolean("fs.automatic.close", false); // See SDC-5451, we set hadoop.treat.subject.external automatically to take advantage of HADOOP-13805 HadoopConfigurationUtils.configureHadoopTreatSubjectExternal(conf); if (hdfsKerberos) { conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION, UserGroupInformation.AuthenticationMethod.KERBEROS.name()); try {/*from ww w .ja v a 2 s . c o m*/ conf.set(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, "hdfs/_HOST@" + HadoopSecurityUtil.getDefaultRealm()); } catch (Exception ex) { if (!hdfsConfigs.stream().anyMatch(i -> DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY.equals(i.key))) { issues.add(context.createConfigIssue(Groups.HADOOP_FS.name(), null, Errors.HADOOPFS_28, ex.toString())); } } } if (hdfsConfDir != null && !hdfsConfDir.isEmpty()) { File hadoopConfigDir = new File(hdfsConfDir); if ((context.getExecutionMode() == ExecutionMode.CLUSTER_BATCH || context.getExecutionMode() == ExecutionMode.CLUSTER_YARN_STREAMING || context.getExecutionMode() == ExecutionMode.CLUSTER_MESOS_STREAMING) && hadoopConfigDir.isAbsolute()) { //Do not allow absolute hadoop config directory in cluster mode issues.add(context.createConfigIssue(Groups.HADOOP_FS.name(), getConfigBeanPrefix() + "hdfsConfDir", Errors.HADOOPFS_45, hdfsConfDir)); } else { if (!hadoopConfigDir.isAbsolute()) { hadoopConfigDir = new File(context.getResourcesDirectory(), hdfsConfDir).getAbsoluteFile(); } if (!hadoopConfigDir.exists()) { issues.add(context.createConfigIssue(Groups.HADOOP_FS.name(), getConfigBeanPrefix() + "hdfsConfDir", Errors.HADOOPFS_25, hadoopConfigDir.getPath())); } else if (!hadoopConfigDir.isDirectory()) { issues.add(context.createConfigIssue(Groups.HADOOP_FS.name(), getConfigBeanPrefix() + "hdfsConfDir", Errors.HADOOPFS_26, hadoopConfigDir.getPath())); } else { File coreSite = new File(hadoopConfigDir, "core-site.xml"); if (coreSite.exists()) { if (!coreSite.isFile()) { issues.add(context.createConfigIssue(Groups.HADOOP_FS.name(), getConfigBeanPrefix() + "hdfsConfDir", Errors.HADOOPFS_27, coreSite.getPath())); } conf.addResource(new Path(coreSite.getAbsolutePath())); } File hdfsSite = new File(hadoopConfigDir, "hdfs-site.xml"); if (hdfsSite.exists()) { if (!hdfsSite.isFile()) { issues.add(context.createConfigIssue(Groups.HADOOP_FS.name(), getConfigBeanPrefix() + "hdfsConfDir", Errors.HADOOPFS_27, hdfsSite.getPath())); } conf.addResource(new Path(hdfsSite.getAbsolutePath())); } } } } else { Optional<HadoopConfigBean> fsDefaultFS = hdfsConfigs.stream() .filter(item -> CommonConfigurationKeys.FS_DEFAULT_NAME_KEY.equals(item.key)).findFirst(); if (StringUtils.isEmpty(hdfsUri) && !fsDefaultFS.isPresent()) { // No URI, no config dir, and no fs.defaultFS config param // Avoid defaulting to writing to file:/// (SDC-5143) issues.add(context.createConfigIssue(Groups.HADOOP_FS.name(), getConfigBeanPrefix() + "hdfsUri", Errors.HADOOPFS_61)); } } for (HadoopConfigBean configBean : hdfsConfigs) { try { conf.set(configBean.key, configBean.value.get()); } catch (StageException e) { issues.add(context.createConfigIssue(Groups.HADOOP_FS.name(), getConfigBeanPrefix() + "hdfsConfigs", Errors.HADOOPFS_62, e.toString())); } } return conf; }
From source file:com.streamsets.pipeline.stage.destination.hdfs.HdfsTarget.java
License:Apache License
Configuration getHadoopConfiguration(List<ConfigIssue> issues) {
Configuration conf = new Configuration();
conf.setClass("fs.file.impl", RawLocalFileSystem.class, FileSystem.class);
if (hdfsKerberos) {
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,
UserGroupInformation.AuthenticationMethod.KERBEROS.name());
try {//from w ww . j a v a 2s. c o m
conf.set(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, "hdfs/_HOST@" + KerberosUtil.getDefaultRealm());
} catch (Exception ex) {
if (!hdfsConfigs.containsKey(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY)) {
issues.add(getContext().createConfigIssue(Groups.HADOOP_FS.name(), null, Errors.HADOOPFS_28,
ex.toString()));
}
}
}
if (hadoopConfDir != null && !hadoopConfDir.isEmpty()) {
File hadoopConfigDir = new File(hadoopConfDir);
if (getContext().isClusterMode() && hadoopConfigDir.isAbsolute()) {
//Do not allow absolute hadoop config directory in cluster mode
issues.add(getContext().createConfigIssue(Groups.HADOOP_FS.name(), "hadoopConfDir",
Errors.HADOOPFS_45, hadoopConfDir));
} else {
if (!hadoopConfigDir.isAbsolute()) {
hadoopConfigDir = new File(getContext().getResourcesDirectory(), hadoopConfDir)
.getAbsoluteFile();
}
if (!hadoopConfigDir.exists()) {
issues.add(getContext().createConfigIssue(Groups.HADOOP_FS.name(), "hadoopConfDir",
Errors.HADOOPFS_25, hadoopConfigDir.getPath()));
} else if (!hadoopConfigDir.isDirectory()) {
issues.add(getContext().createConfigIssue(Groups.HADOOP_FS.name(), "hadoopConfDir",
Errors.HADOOPFS_26, hadoopConfigDir.getPath()));
} else {
File coreSite = new File(hadoopConfigDir, "core-site.xml");
if (coreSite.exists()) {
if (!coreSite.isFile()) {
issues.add(getContext().createConfigIssue(Groups.HADOOP_FS.name(), "hadoopConfDir",
Errors.HADOOPFS_27, coreSite.getPath()));
}
conf.addResource(new Path(coreSite.getAbsolutePath()));
}
File hdfsSite = new File(hadoopConfigDir, "hdfs-site.xml");
if (hdfsSite.exists()) {
if (!hdfsSite.isFile()) {
issues.add(getContext().createConfigIssue(Groups.HADOOP_FS.name(), "hadoopConfDir",
Errors.HADOOPFS_27, hdfsSite.getPath()));
}
conf.addResource(new Path(hdfsSite.getAbsolutePath()));
}
}
}
}
for (Map.Entry<String, String> config : hdfsConfigs.entrySet()) {
conf.set(config.getKey(), config.getValue());
}
return conf;
}
From source file:com.streamsets.pipeline.stage.destination.hdfs.HdfsTargetConfigBean.java
License:Apache License
private Configuration getHadoopConfiguration(Stage.Context context, List<Stage.ConfigIssue> issues) { Configuration conf = new Configuration(); conf.setClass("fs.file.impl", RawLocalFileSystem.class, FileSystem.class); if (hdfsKerberos) { conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION, UserGroupInformation.AuthenticationMethod.KERBEROS.name()); try {//from ww w .j a v a2 s. c om conf.set(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, "hdfs/_HOST@" + HadoopSecurityUtil.getDefaultRealm()); } catch (Exception ex) { if (!hdfsConfigs.containsKey(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY)) { issues.add(context.createConfigIssue(Groups.HADOOP_FS.name(), null, Errors.HADOOPFS_28, ex.toString())); } } } if (hdfsConfDir != null && !hdfsConfDir.isEmpty()) { File hadoopConfigDir = new File(hdfsConfDir); if ((context.getExecutionMode() == ExecutionMode.CLUSTER_BATCH || context.getExecutionMode() == ExecutionMode.CLUSTER_YARN_STREAMING || context.getExecutionMode() == ExecutionMode.CLUSTER_MESOS_STREAMING) && hadoopConfigDir.isAbsolute()) { //Do not allow absolute hadoop config directory in cluster mode issues.add(context.createConfigIssue(Groups.HADOOP_FS.name(), HDFS_TARGET_CONFIG_BEAN_PREFIX + "hdfsConfDir", Errors.HADOOPFS_45, hdfsConfDir)); } else { if (!hadoopConfigDir.isAbsolute()) { hadoopConfigDir = new File(context.getResourcesDirectory(), hdfsConfDir).getAbsoluteFile(); } if (!hadoopConfigDir.exists()) { issues.add(context.createConfigIssue(Groups.HADOOP_FS.name(), HDFS_TARGET_CONFIG_BEAN_PREFIX + "hdfsConfDir", Errors.HADOOPFS_25, hadoopConfigDir.getPath())); } else if (!hadoopConfigDir.isDirectory()) { issues.add(context.createConfigIssue(Groups.HADOOP_FS.name(), HDFS_TARGET_CONFIG_BEAN_PREFIX + "hdfsConfDir", Errors.HADOOPFS_26, hadoopConfigDir.getPath())); } else { File coreSite = new File(hadoopConfigDir, "core-site.xml"); if (coreSite.exists()) { if (!coreSite.isFile()) { issues.add(context.createConfigIssue(Groups.HADOOP_FS.name(), HDFS_TARGET_CONFIG_BEAN_PREFIX + "hdfsConfDir", Errors.HADOOPFS_27, coreSite.getPath())); } conf.addResource(new Path(coreSite.getAbsolutePath())); } File hdfsSite = new File(hadoopConfigDir, "hdfs-site.xml"); if (hdfsSite.exists()) { if (!hdfsSite.isFile()) { issues.add(context.createConfigIssue(Groups.HADOOP_FS.name(), HDFS_TARGET_CONFIG_BEAN_PREFIX + "hdfsConfDir", Errors.HADOOPFS_27, hdfsSite.getPath())); } conf.addResource(new Path(hdfsSite.getAbsolutePath())); } } } } for (Map.Entry<String, String> config : hdfsConfigs.entrySet()) { conf.set(config.getKey(), config.getValue()); } return conf; }
From source file:com.streamsets.pipeline.stage.destination.hdfs.writer.TestRecordWriter.java
License:Apache License
private FileSystem getRawLocalFileSystem() throws Exception { Configuration conf = new Configuration(); conf.setClass("fs.file.impl", RawLocalFileSystem.class, FileSystem.class); return FileSystem.get(new URI("file:///"), conf); }
From source file:com.thinkbiganalytics.kylo.catalog.aws.S3FileSystemProviderTest.java
License:Apache License
/** * Verify listing buckets using the s3a scheme. *//* w w w. j a v a2 s . co m*/ @Test @SuppressWarnings("unchecked") public void listFilesS3a() { // Setup configuration final Configuration conf = new Configuration(false); conf.setClass(Constants.S3_CLIENT_FACTORY_IMPL, MockS3ClientFactory.class, S3ClientFactory.class); // Test listing buckets final S3FileSystemProvider provider = new S3FileSystemProvider(); final List<DataSetFile> files = provider.listFiles(new Path(S3A), conf); Assert.assertThat(files, CoreMatchers.hasItems(isDataSetFile(MockS3ClientFactory.BUCKET1), isDataSetFile(MockS3ClientFactory.BUCKET2))); Assert.assertEquals(2, files.size()); }
From source file:com.thinkbiganalytics.kylo.catalog.aws.S3FileSystemProviderTest.java
License:Apache License
/** * Verify creating a custom credentials provider. *//* w w w . j ava 2 s .c o m*/ @Test @SuppressWarnings("squid:S3655") public void getCustomCredentialsProvider() { // Test empty configuration final S3FileSystemProvider provider = new S3FileSystemProvider(); Assert.assertFalse("Expected empty credentials", provider.getCustomCredentialsProvider(S3N, new Configuration(false)).isPresent()); // Test with custom credentials provider final Configuration conf = new Configuration(false); conf.setClass("fs.s3.customAWSCredentialsProvider", EnvironmentVariableCredentialsProvider.class, AWSCredentialsProvider.class); final Optional<AWSCredentialsProvider> credentials = provider.getCustomCredentialsProvider(S3N, conf); Assert.assertTrue("Expected custom credentials", credentials.isPresent()); Assert.assertTrue("Expected EnvironmentVariableCredentialsProvider", credentials.get() instanceof EnvironmentVariableCredentialsProvider); }
From source file:com.thinkbiganalytics.kylo.hadoop.FileSystemUtil.java
License:Apache License
/** * Adds Hadoop {@link FileSystem} classes to the Hadoop configuration. *///w w w . j a va 2 s . com public static void registerFileSystems(@Nonnull final Iterable<FileSystem> fileSystems, @Nonnull final Configuration conf) { for (final FileSystem fs : fileSystems) { try { final String scheme = fs.getScheme(); final Class clazz = fs.getClass(); log.debug("Found {} FileSystem using class: {}", scheme, clazz); conf.setClass("fs." + scheme + ".impl", clazz, FileSystem.class); } catch (final Exception e) { log.warn("Cannot load FileSystem using class: {}: {}", fs.getClass().getName(), e, e); } } }