List of usage examples for org.apache.hadoop.conf Configuration setInt
public void setInt(String name, int value)
name property to an int. From source file:com.skp.experiment.cf.als.hadoop.ParallelALSFactorizationJob.java
License:Apache License
private void runSolver(Path ratings, Path output, Path pathToUorI, Path pathToTranspose, int numRows, boolean largeMatrix) throws ClassNotFoundException, IOException, InterruptedException { @SuppressWarnings("rawtypes") Class<? extends Mapper> solverMapper = implicitFeedback ? SolveImplicitFeedbackMultithreadedMapper.class : SolveExplicitFeedbackMapper.class; Job solverForUorI = prepareJob(ratings, output, SequenceFileInputFormat.class, solverMapper, IntWritable.class, VectorWritable.class, SequenceFileOutputFormat.class); Configuration solverConf = solverForUorI.getConfiguration(); long matrixSizeExp = (long) (8L * numRows * numFeatures * SAFE_MARGIN); long memoryThreshold = HadoopClusterUtil.PHYSICAL_MEMERY_LIMIT / HadoopClusterUtil.MAP_TASKS_PER_NODE; int numTaskPerDataNode = Math.max(1, (int) (HadoopClusterUtil.PHYSICAL_MEMERY_LIMIT / matrixSizeExp)); if (matrixSizeExp > memoryThreshold) { solverConf.set("mapred.child.java.opts", "-Xmx8g"); solverConf.set("mapred.map.child.java.opts", "-Xmx8g"); solverConf.setLong("dfs.block.size", HadoopClusterUtil.getMaxBlockSize(getConf(), pathToTransformed())); solverConf.setInt("mapred.map.tasks", HadoopClusterUtil.getNumberOfTaskTrackers(getConf())); solverConf.setLong("mapred.min.split.size", HadoopClusterUtil.getMaxBlockSize(getConf(), pathToTransformed())); solverConf.setLong("mapred.max.split.size", HadoopClusterUtil.getMaxBlockSize(getConf(), pathToTransformed())); solverConf.set(SolveImplicitFeedbackMultithreadedMapper.LOCK_FILE, pathToHostLocks().toString()); solverConf.setInt(SolveImplicitFeedbackMultithreadedMapper.LOCK_FILE_NUMS, Math.min(HadoopClusterUtil.MAP_TASKS_PER_NODE, numTaskPerDataNode)); } else {//from ww w.ja v a2 s .co m solverConf.setLong("mapred.min.split.size", HadoopClusterUtil.getMinInputSplitSizeMax(getConf(), ratings)); solverConf.setLong("mapred.max.split.size", HadoopClusterUtil.getMinInputSplitSizeMax(getConf(), ratings)); solverConf.setInt("mapred.map.tasks", HadoopClusterUtil.getNumberOfTaskTrackers(getConf()) * multiplyMapTasks); //solverConf.setBoolean("mapred.map.tasks.speculative.execution", false); } solverConf.setLong("mapred.task.timeout", taskTimeout); solverConf.setBoolean("mapred.map.tasks.speculative.execution", false); solverConf.set(SolveImplicitFeedbackMultithreadedMapper.LAMBDA, String.valueOf(lambda)); solverConf.set(SolveImplicitFeedbackMultithreadedMapper.ALPHA, String.valueOf(alpha)); solverConf.setInt(SolveImplicitFeedbackMultithreadedMapper.NUM_FEATURES, numFeatures); solverConf.setInt(SolveImplicitFeedbackMultithreadedMapper.NUM_ROWS, numRows); solverConf.set(SolveImplicitFeedbackMultithreadedMapper.FEATURE_MATRIX, pathToUorI.toString()); solverConf.set(SolveImplicitFeedbackMultithreadedMapper.FEATURE_MATRIX_TRANSPOSE, pathToTranspose.toString()); solverForUorI.waitForCompletion(true); }
From source file:com.splicemachine.access.hbase.HBaseConnectionFactory.java
License:Apache License
private HBaseConnectionFactory(SConfiguration configuration) { this.config = configuration; this.namespace = configuration.getNamespace(); this.namespaceBytes = Bytes.toBytes(namespace); try {//w w w . j a v a2 s . co m Configuration config = (Configuration) configuration.getConfigSource().unwrapDelegate(); this.connection = ConnectionFactory.createConnection(config); Configuration clonedConfig = new Configuration(config); clonedConfig.setInt("hbase.client.retries.number", 1); this.noRetryConnection = ConnectionFactory.createConnection(clonedConfig); } catch (IOException ioe) { throw new RuntimeException(ioe); } }
From source file:com.splicemachine.orc.OrcConf.java
License:Open Source License
public static void setIntVar(Configuration conf, OrcConf.ConfVars var, int val) { conf.setInt(var.varname, val); }
From source file:com.splicemachine.si.testsetup.HBaseSITestEnv.java
License:Apache License
private void startCluster(Configuration conf) throws Exception { int basePort = getNextBasePort(); // -> MapR work-around conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); conf.set("fs.default.name", "file:///"); conf.set("fs.hdfs.client", "org.apache.hadoop.hdfs.DistributedFileSystem"); System.setProperty("zookeeper.sasl.client", "false"); System.setProperty("zookeeper.sasl.serverconfig", "fake"); // <- MapR work-around conf.setInt("hbase.master.port", basePort); conf.setInt("hbase.master.info.port", basePort + 1); conf.setInt("hbase.regionserver.port", basePort + 2); conf.setInt("hbase.regionserver.info.port", basePort + 3); testUtility = new HBaseTestingUtility(conf); Configuration configuration = testUtility.getConfiguration(); // -> MapR work-around configuration.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); configuration.set("fs.default.name", "file:///"); configuration.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); configuration.set("fs.hdfs.client", "org.apache.hadoop.hdfs.DistributedFileSystem"); System.setProperty("zookeeper.sasl.client", "false"); System.setProperty("zookeeper.sasl.serverconfig", "fake"); // <- MapR work-around configuration.setInt("hbase.master.port", basePort); configuration.setInt("hbase.master.info.port", basePort + 1); configuration.setInt("hbase.regionserver.port", basePort + 2); configuration.setInt("hbase.regionserver.info.port", basePort + 3); if (FileSystem.class.getProtectionDomain().getCodeSource().getLocation().getPath().contains("mapr")) { testUtility.startMiniCluster(1); } else {//from www. j a v a2 s . c o m testUtility.startMiniZKCluster(); testUtility.startMiniHBaseCluster(1, 1, null, null, false); } ZkUtils.getZkManager().initialize(HConfiguration.getConfiguration()); ZkUtils.initializeZookeeper(); }
From source file:com.splicemachine.test.SpliceTestPlatformConfig.java
License:Apache License
public static Configuration create(String hbaseRootDirUri, Integer masterPort, Integer masterInfoPort, Integer regionServerPort, Integer regionServerInfoPort, Integer derbyPort, boolean failTasksRandomly) { Configuration config = HConfiguration.unwrapDelegate(); config.set(SQLConfiguration.STORAGE_FACTORY_HOME, hbaseRootDirUri); ////from w w w .ja va2s . c o m // Coprocessors // config.set("hbase.coprocessor.regionserver.classes", getRegionServerCoprocessorsAsString()); config.set("hbase.coprocessor.region.classes", getRegionCoprocessorsAsString()); config.set("hbase.coprocessor.master.classes", getMasterCoprocessorsAsString()); // // Networking // config.set("hbase.zookeeper.quorum", "127.0.0.1:2181"); config.setInt("hbase.master.port", masterPort); config.setInt("hbase.master.info.port", masterInfoPort); config.setInt("hbase.regionserver.port", regionServerPort); config.setInt("hbase.regionserver.info.port", regionServerInfoPort); config.setInt("hbase.master.jmx.port", HConfiguration.DEFAULT_JMX_BIND_PORT); // this is set because the HBase master and regionserver are running on the same machine and in the same JVM config.setInt(SQLConfiguration.NETWORK_BIND_PORT, derbyPort); config.setClass(DefaultStoreEngine.DEFAULT_COMPACTOR_CLASS_KEY, SpliceDefaultCompactor.class, Compactor.class); // config.setClass(ConsistencyControlUtils.MVCC_IMPL, SIMultiVersionConsistencyControl.class, ConsistencyControl.class); config.setClass(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, SpliceDefaultCompactionPolicy.class, CompactionPolicy.class); // // Networking -- interfaces // // force use of loop back interface on MacOSX, else don't set it // if (System.getProperty("os.name").contains("Mac") ) { // String interfaceName = "lo0"; // config.set("hbase.zookeeper.dns.interface", interfaceName); // config.set("hbase.master.dns.interface", interfaceName); // config.set("hbase.regionserver.dns.interface", interfaceName); // } // // File System // config.set("fs.defaultFS", "file:///"); // MapR Hack, tells it local filesystem // fs.default.name is deprecated config.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); config.setDouble("yarn.nodemanager.resource.io-spindles", 2.0); config.set("fs.default.name", "file:///"); config.set("yarn.nodemanager.container-executor.class", "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor"); // Must allow Cygwin instance to config its own rootURI if (!"CYGWIN".equals(hbaseRootDirUri)) { config.set("hbase.rootdir", hbaseRootDirUri); } // // Threads, timeouts // config.setLong("hbase.rpc.timeout", MINUTES.toMillis(2)); config.setLong("hbase.client.scanner.timeout.period", MINUTES.toMillis(2)); // hbase.regionserver.lease.period is deprecated config.setLong("hbase.client.operation.timeout", MINUTES.toMillis(2)); config.setLong("hbase.regionserver.handler.count", 200); config.setLong("hbase.regionserver.msginterval", 1000); config.setLong("hbase.master.event.waiting.time", 20); config.setLong("hbase.master.lease.thread.wakefrequency", SECONDS.toMillis(3)); // config.setBoolean("hbase.master.loadbalance.bytable",true); config.setInt("hbase.balancer.period", 5000); config.setLong("hbase.server.thread.wakefrequency", SECONDS.toMillis(1)); config.setLong("hbase.client.pause", 100); // // Compaction Controls // config.setLong("hbase.hstore.compaction.min", 5); // min number of eligible files before we compact config.setLong("hbase.hstore.compaction.max", 10); // max files to be selected for a single minor compaction config.setLong("hbase.hstore.compaction.min.size", 16 * MiB); // store files smaller than this will always be eligible for minor compaction. HFiles this size or larger are evaluated by hbase.hstore.compaction.ratio to determine if they are eligible config.setLong("hbase.hstore.compaction.max.size", 248 * MiB); // store files larger than this will be excluded from compaction config.setFloat("hbase.hstore.compaction.ratio", 1.25f); // default is 1.2f, at one point we had this set to 0.25f and 25f (which was likely a typo) // // Memstore, store files, splits // config.setLong(HConstants.HREGION_MAX_FILESIZE, 32 * MiB); // hbase.hregion.max.filesize config.setLong("hbase.hregion.memstore.flush.size", 128 * MiB); // was 512 MiB config.setLong("hbase.hregion.memstore.block.multiplier", 4); config.setFloat("hbase.regionserver.global.memstore.size", 0.25f); // set mem store to 25% of heap config.setLong("hbase.hstore.blockingStoreFiles", 20); // config.set("hbase.regionserver.region.split.policy", "org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy"); // change default split policy. this makes more sense for a standalone/single regionserver // Support SI //config.setClass(HConstants.MVCC_IMPL, SIMultiVersionConsistencyControl.class, ConsistencyControl.class); // // HFile // config.setInt("hfile.index.block.max.size", 16 * 1024); // 16KiB config.setFloat("hfile.block.cache.size", 0.25f); // set block cache to 25% of heap config.setFloat("io.hfile.bloom.error.rate", (float) 0.005); config.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, true); // hfile.block.bloom.cacheonwrite //config.set("hbase.master.hfilecleaner.plugins", getHFileCleanerAsString()); config.set("hbase.master.hfilecleaner.plugins", getHFileCleanerAsString()); // // Misc // config.set("hbase.cluster.distributed", "true"); // don't start zookeeper for us config.set("hbase.master.distributed.log.splitting", "false"); // TODO: explain why we are setting this // AWS Credentials for test... // config.set(ACCESS_KEY, "AKIAJ6HBMCK5ALHVBFPQ"); config.set(SECRET_KEY, "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"); // // Splice // config.setLong("splice.ddl.drainingWait.maximum", SECONDS.toMillis(15)); // wait 15 seconds before bailing on bad ddl statements config.setLong("splice.ddl.maxWaitSeconds", 120000); // // Snapshots // config.setBoolean("hbase.snapshot.enabled", true); HConfiguration.reloadConfiguration(config); return HConfiguration.unwrapDelegate(); }
From source file:com.streamsets.pipeline.stage.destination.mapreduce.MapReduceExecutor.java
License:Apache License
@Override public void write(Batch batch) throws StageException { EvalContext eval = new EvalContext(getContext()); Iterator<Record> it = batch.getRecords(); while (it.hasNext()) { final Record record = it.next(); eval.setRecord(record);/*from w w w.ja va 2 s .c o m*/ Job job = null; try { // Job configuration object is a clone of the original one that we're keeping in mapReduceConfig class final Configuration jobConfiguration = new Configuration(mapReduceConfig.getConfiguration()); // Evaluate all dynamic properties and store them in the configuration job for (Map.Entry<String, String> entry : jobConfig.jobConfigs.entrySet()) { String key = eval.evaluateToString("jobConfigs", entry.getKey(), true); String value = eval.evaluateToString("jobConfigs", entry.getValue(), false); jobConfiguration.set(key, value); } // For build-in job creators, evaluate their properties and persist them in the MR config switch (jobConfig.jobType) { case AVRO_PARQUET: jobConfiguration.set(AvroConversionCommonConstants.INPUT_FILE, eval .evaluateToString("inputFile", jobConfig.avroConversionCommonConfig.inputFile, true)); jobConfiguration.set(AvroConversionCommonConstants.OUTPUT_DIR, eval.evaluateToString( "outputDirectory", jobConfig.avroConversionCommonConfig.outputDirectory, true)); jobConfiguration.setBoolean(AvroConversionCommonConstants.KEEP_INPUT_FILE, jobConfig.avroConversionCommonConfig.keepInputFile); jobConfiguration.set(AvroParquetConstants.COMPRESSION_CODEC_NAME, eval.evaluateToString( "compressionCodec", jobConfig.avroParquetConfig.compressionCodec, false)); jobConfiguration.setInt(AvroParquetConstants.ROW_GROUP_SIZE, jobConfig.avroParquetConfig.rowGroupSize); jobConfiguration.setInt(AvroParquetConstants.PAGE_SIZE, jobConfig.avroParquetConfig.pageSize); jobConfiguration.setInt(AvroParquetConstants.DICTIONARY_PAGE_SIZE, jobConfig.avroParquetConfig.dictionaryPageSize); jobConfiguration.setInt(AvroParquetConstants.MAX_PADDING_SIZE, jobConfig.avroParquetConfig.maxPaddingSize); jobConfiguration.setBoolean(AvroConversionCommonConstants.OVERWRITE_TMP_FILE, jobConfig.avroConversionCommonConfig.overwriteTmpFile); break; case AVRO_ORC: jobConfiguration.set(AvroConversionCommonConstants.INPUT_FILE, eval .evaluateToString("inputFile", jobConfig.avroConversionCommonConfig.inputFile, true)); jobConfiguration.set(AvroConversionCommonConstants.OUTPUT_DIR, eval.evaluateToString( "outputDirectory", jobConfig.avroConversionCommonConfig.outputDirectory, true)); jobConfiguration.setBoolean(AvroConversionCommonConstants.KEEP_INPUT_FILE, jobConfig.avroConversionCommonConfig.keepInputFile); jobConfiguration.setBoolean(AvroConversionCommonConstants.OVERWRITE_TMP_FILE, jobConfig.avroConversionCommonConfig.overwriteTmpFile); jobConfiguration.setInt(AvroOrcConstants.ORC_BATCH_SIZE, jobConfig.avroOrcConfig.orcBatchSize); break; case CUSTOM: // Nothing because custom is generic one that have no special config properties break; default: throw new UnsupportedOperationException("Unsupported JobType: " + jobConfig.jobType); } job = createAndSubmitJob(jobConfiguration); } catch (IOException | InterruptedException | ELEvalException e) { LOG.error("Can't submit mapreduce job", e); errorRecordHandler.onError( new OnRecordErrorException(record, MapReduceErrors.MAPREDUCE_0005, e.getMessage(), e)); } if (job != null) { MapReduceExecutorEvents.JOB_CREATED.create(getContext()).with("tracking-url", job.getTrackingURL()) .with("job-id", job.getJobID().toString()).createAndSend(); } } }
From source file:com.streamsets.pipeline.stage.processor.hbase.HBaseProcessorIT.java
License:Apache License
private Configuration getHBaseConfiguration(HBaseLookupConfig config) { Configuration hbaseConfig = HBaseUtil.getHBaseConfiguration(new ArrayList<Stage.ConfigIssue>(), context, Groups.HBASE.getLabel(), config.hBaseConnectionConfig.hbaseConfDir, config.hBaseConnectionConfig.zookeeperQuorum, config.hBaseConnectionConfig.zookeeperParentZnode, config.hBaseConnectionConfig.clientPort, config.hBaseConnectionConfig.tableName, config.hBaseConnectionConfig.kerberosAuth, config.hBaseConnectionConfig.hbaseConfigs); HBaseUtil.setIfNotNull(hbaseConfig, HConstants.ZOOKEEPER_QUORUM, config.hBaseConnectionConfig.zookeeperQuorum); hbaseConfig.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, config.hBaseConnectionConfig.clientPort); HBaseUtil.setIfNotNull(hbaseConfig, HConstants.ZOOKEEPER_ZNODE_PARENT, config.hBaseConnectionConfig.zookeeperParentZnode); return hbaseConfig; }
From source file:com.streamsets.pipeline.stage.processor.transformer.WholeFileTransformerProcessor.java
License:Apache License
/** * Convert Avro record to Parquet//w w w . j av a 2s .c o m * @param sourceFileName the source Avro file name * @param fileReader the {@link org.apache.avro.file.DataFileStream} Avro file reader * @param tempParquetFile the {@link java.nio.file.Path} temporary parquet file path */ private void writeParquet(String sourceFileName, DataFileStream<GenericRecord> fileReader, Path tempParquetFile) throws StageException { long recordCount = 0; GenericRecord avroRecord; Schema schema = fileReader.getSchema(); LOG.debug("Start reading input file : {}", sourceFileName); try { // initialize parquet writer Configuration jobConfiguration = new Configuration(); String compressionCodecName = compressionElEval.eval(variables, jobConfig.avroParquetConfig.compressionCodec, String.class); jobConfiguration.set(AvroParquetConstants.COMPRESSION_CODEC_NAME, compressionCodecName); jobConfiguration.setInt(AvroParquetConstants.ROW_GROUP_SIZE, jobConfig.avroParquetConfig.rowGroupSize); jobConfiguration.setInt(AvroParquetConstants.PAGE_SIZE, jobConfig.avroParquetConfig.pageSize); jobConfiguration.setInt(AvroParquetConstants.DICTIONARY_PAGE_SIZE, jobConfig.avroParquetConfig.dictionaryPageSize); jobConfiguration.setInt(AvroParquetConstants.MAX_PADDING_SIZE, jobConfig.avroParquetConfig.maxPaddingSize); // Parquet writer ParquetWriter.Builder builder = AvroToParquetConverterUtil.initializeWriter( new org.apache.hadoop.fs.Path(tempParquetFile.toString()), schema, jobConfiguration); parquetWriter = builder.build(); while (fileReader.hasNext()) { avroRecord = fileReader.next(); parquetWriter.write(avroRecord); recordCount++; } parquetWriter.close(); } catch (IOException ex) { throw new TransformerStageCheckedException(Errors.CONVERT_08, sourceFileName, recordCount, ex); } LOG.debug("Finished writing {} records to {}", recordCount, tempParquetFile.getFileName()); }
From source file:com.syscrest.oozie.graphite.GraphiteMRCounterExecutorExecutorTest.java
License:Apache License
@Before public void setUp() throws Exception { System.setProperty(Services.OOZIE_HOME_DIR, "/tmp"); final Configuration oozieConf = new Configuration(); oozieConf.setInt(ActionExecutor.MAX_RETRIES, 1); fixture = new GraphiteMRCounterExecutor() { @Override/* w w w .j av a 2s. c om*/ public Configuration getOozieConf() { return oozieConf; } }; }
From source file:com.taobao.adfs.database.MysqlServerController.java
License:Apache License
public void formatData(Configuration conf) throws Throwable { Utilities.logInfo(logger, "mysql server is formatting"); setMysqlDefaultConf(conf);/*from w ww . ja v a 2s.co m*/ // stop mysql server and initialize data String mysqlServerPid = stopServer(conf); backupData(conf); String mysqlDataPath = Utilities.getNormalPath(conf.get("mysql.server.data.path", ".")); Utilities.mkdirs(mysqlDataPath, true); String commandForCreateMysqlData = "mysql_install_db"; commandForCreateMysqlData += " --force"; commandForCreateMysqlData += " --general_log"; commandForCreateMysqlData += " --no-defaults"; commandForCreateMysqlData += " --basedir=" + getMysqlConf(conf, "mysqld.basedir"); commandForCreateMysqlData += " --datadir=" + mysqlDataPath; Utilities.logInfo(logger, "mysql server is installing new data, command=", commandForCreateMysqlData); Utilities.runCommand(commandForCreateMysqlData, 0, conf.get("mysql.server.bin.path"), null); Utilities.logInfo(logger, "mysql server has installed new data"); // start mysql server and set access control startServer(conf); String commandForSetMysqlAccess = "mysql -uroot"; commandForSetMysqlAccess += " --socket=" + getMysqlConf(conf, "mysqld.socket"); // commandForSetMysqlServerPassword += " password '" + conf.get("mysql.password", "root") + "'"; commandForSetMysqlAccess += " --execute=\""; commandForSetMysqlAccess += "use mysql;delete from user;grant all privileges on *.* to 'root'@'%' identified by 'root';flush privileges;"; commandForSetMysqlAccess += "\""; Utilities.logInfo(logger, "mysql server is setting privileges, command=", commandForSetMysqlAccess); Utilities.runCommand(commandForSetMysqlAccess, 0, conf.get("mysql.server.bin.path"), getMysqlLibPath(conf.get("mysql.server.bin.path"))); Utilities.logInfo(logger, "mysql server has set privileges"); // create database try { createDatabase(conf); } catch (Throwable t) { int retryIndex = conf.getInt("mysql.server.format.retry.index", 0); if (retryIndex >= conf.getInt("mysql.server.format.retry.max", 3)) throw new IOException(t); conf.setInt("mysql.server.format.retry.index", ++retryIndex); Utilities.logError(logger, "mysql server fails to create database, retryIndex=", retryIndex, t); formatData(conf); return; } // restore mysql server status before format if (mysqlServerPid.isEmpty() && conf.getBoolean("mysql.server.restore", false)) stopServer(conf); Utilities.logInfo(logger, "mysql server is formatted"); }