List of usage examples for org.apache.hadoop.conf Configuration Configuration
public Configuration()
From source file:azure.TweetUpload.java
License:Apache License
public static void main(String[] args) { try { String filePath = "hdfs://localhost.localdomain:8020/tmp/hive-mapred/" + args[0] + "/000000_0"; // File location Configuration configuration = new Configuration(); Path path = new Path(filePath); Path newFilePath = new Path("temp_" + args[0]); FileSystem fs = path.getFileSystem(configuration); // w ww . j av a 2 s . c o m fs.copyToLocalFile(path, newFilePath); // Copy temporary to local directory CloudStorageAccount account = CloudStorageAccount .parse(storageConnectionString); CloudBlobClient serviceClient = account.createCloudBlobClient(); CloudBlobContainer container = serviceClient .getContainerReference("container_name_here"); // Container name (must be lower case) container.createIfNotExists(); // Upload file CloudBlockBlob blob = container .getBlockBlobReference("user/rdp_username_here/analysisFiles/" + args[0] + ".tsv"); File sourceFile = new File(newFilePath.toString()); blob.upload(new FileInputStream(sourceFile), sourceFile.length()); File tmpFile = new File(newFilePath.toString()); tmpFile.delete(); // Delete the temporary file // In case of errors } catch (Exception e) { System.exit(-1); } }
From source file:backup.integration.MiniClusterTestBase.java
License:Apache License
private Configuration setupConfig(File hdfsDir) throws Exception { Configuration conf = new Configuration(); File backup = new File(tmpHdfs, "backup"); backup.mkdirs();/*from w w w .j av a 2s . c o m*/ conf.set(DFS_BACKUP_NAMENODE_LOCAL_DIR_KEY, backup.getAbsolutePath()); conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, hdfsDir.getAbsolutePath()); conf.set(DFSConfigKeys.DFS_DATANODE_FSDATASET_FACTORY_KEY, BackupFsDatasetSpiFactory.class.getName()); conf.set(DFSConfigKeys.DFS_DATANODE_PLUGINS_KEY, DataNodeBackupServicePlugin.class.getName()); conf.set(DFSConfigKeys.DFS_NAMENODE_PLUGINS_KEY, NameNodeBackupServicePlugin.class.getName()); conf.setInt(BackupConstants.DFS_BACKUP_DATANODE_RPC_PORT_KEY, 0); conf.setInt(BackupConstants.DFS_BACKUP_NAMENODE_HTTP_PORT_KEY, 0); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 2);// 3 conf.setLong(DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_MINIMUM_INTERVAL_KEY, 2);// 3 conf.setLong(DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY, 6000);// 30000 conf.setLong(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 6000);// 5*60*1000 org.apache.commons.configuration.Configuration configuration = BackupUtil.convert(conf); setupBackupStore(configuration); @SuppressWarnings("unchecked") Iterator<String> keys = configuration.getKeys(); while (keys.hasNext()) { String key = keys.next(); conf.set(key, configuration.getString(key)); } return conf; }
From source file:backup.store.ExternalExtendedBlockSort.java
License:Apache License
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Path dir = new Path("file:///home/apm/Development/git-projects/hdfs-backup/hdfs-backup-core/tmp"); dir.getFileSystem(conf).delete(dir, true); long start = System.nanoTime(); try (ExternalExtendedBlockSort<LongWritable> sort = new ExternalExtendedBlockSort<>(conf, dir, LongWritable.class)) { Random random = new Random(); for (int bp = 0; bp < 1; bp++) { String bpid = UUID.randomUUID().toString(); for (int i = 0; i < 10000000; i++) { // for (int i = 0; i < 10; i++) { long genstamp = random.nextInt(20000); long blockId = random.nextLong(); ExtendedBlock extendedBlock = new ExtendedBlock(bpid, blockId, random.nextInt(Integer.MAX_VALUE), genstamp); sort.add(extendedBlock, new LongWritable(blockId)); }// w w w.j a va2 s .c o m } System.out.println("finished"); sort.finished(); System.out.println("interate"); for (String blockPoolId : sort.getBlockPoolIds()) { ExtendedBlockEnum<LongWritable> blockEnum = sort.getBlockEnum(blockPoolId); ExtendedBlock block; long l = 0; while ((block = blockEnum.next()) != null) { // System.out.println(block); long blockId = block.getBlockId(); l += blockId; LongWritable currentValue = blockEnum.currentValue(); if (currentValue.get() != blockId) { System.err.println("Error " + blockId); } } System.out.println(l); } } long end = System.nanoTime(); System.out.println("Time [" + (end - start) / 1000000.0 + " ms]"); }
From source file:batch.BatchScan2Html.java
License:Apache License
public static void writeAccumuloTableToHdfsAsHtml() throws IOException, URISyntaxException { Configuration configuration = new Configuration(); //TODO add options for URI and output Path FileSystem hdfs = FileSystem.get(new URI("hdfs://n001:54310"), configuration); Path file = new Path("hdfs://n001:54310/s2013/batch/table.html"); //TODO add option to override file default: true if (hdfs.exists(file)) { hdfs.delete(file, true);// w w w. j ava2s. c o m } startTime = System.currentTimeMillis(); OutputStream os = hdfs.create(file, new Progressable() { public void progress() { // TODO add a better progress descriptor crudeRunTime = System.currentTimeMillis() - startTime; out.println("...bytes written: [ " + bytesWritten + " ]"); out.println("...bytes / second: [ " + (bytesWritten / crudeRunTime) * 1000 + " ]"); } }); BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8")); // TODO add option for table id { example } writeHtmlTableHeader(br, "example", new ArrayList<String>(Arrays.asList("Row ID", "Column Family", "Column Qualifier", "Column Visibility", "Timestamp", "Value"))); writeHtmlTableBody(br); out.println("Total bytes written: " + bytesWritten); out.println("Total crude time: " + crudeRunTime / 1000); br.close(); hdfs.close(); }
From source file:bb.BranchAndBound.java
License:Apache License
static Job getJob(String input, String output, String dataDir, int iteration) throws Exception { Configuration conf = new Configuration(); FileSystem hdfs = FileSystem.get(conf); FileStatus[] fileStatus = hdfs.listStatus(new Path(input)); for (int i = 0; i < fileStatus.length; ++i) { if (fileStatus[i].getLen() == 0) { hdfs.delete(fileStatus[i].getPath()); }/* ww w . j a v a2 s. c o m*/ } DistributedCache.addCacheFile(new URI(dataDir + "/data"), conf); Job ret = new Job(conf, dataDir + "_iteration_" + iteration); ret.setJarByClass(BranchAndBound.class); ret.setMapperClass(BBMapper1.class); ret.setReducerClass(BBReducer.class); //ret.setReducerClass(MergeReducer.class); FileInputFormat.setInputPaths(ret, new Path(input)); //if( iteration > 7 ) FileInputFormat.setMinInputSplitSize(ret, 67108864); FileOutputFormat.setOutputPath(ret, new Path(output)); ret.setOutputKeyClass(NullWritable.class); ret.setOutputValueClass(Text.class); return ret; }
From source file:bb.BranchAndBound.java
License:Apache License
public static void main(String[] args) throws Exception { /*Configuration conf = new Configuration(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if (otherArgs.length != 2) {/* w w w .j ava 2s.c o m*/ System.err.println("Usage: branchandbound <input> <output>"); System.exit(2); } Job job = new Job(conf, "branch and bound"); job.setJarByClass(BranchAndBound.class); job.setMapperClass(BBMapper.class); // job.setCombinerClass(IntSumReducer.class); // job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); FileInputFormat.addInputPath(job, new Path(otherArgs[0])); FileOutputFormat.setOutputPath(job, new Path(otherArgs[1])); System.exit(job.waitForCompletion(true) ? 0 : 1);*/ int n; String[] inputargs = new GenericOptionsParser(new Configuration(), args).getRemainingArgs(); if (inputargs.length != 2) { System.err.println("Usage: branchandbound <data directory> <n>"); System.exit(2); } n = Integer.parseInt(inputargs[1]); String dataDir = inputargs[0]; String prev_output = dataDir + "/input"; /* for( int i = 1 ; i <= n ; i++ ) { for( int j = 0 ; j < 2 ; j++ ) { String input = prev_output ; String output = inputargs[1] + "/iteration" + i + "_" + j ; Job job = getJob(input, output, i, j) ; job.waitForCompletion(true) ; // if failed ???? prev_output = output; } } */ //prev_output = dataDir + "/output" + "/iteration" + 17; long totalNodes = 0; long searchedNodes = 0; long cutbyDEE = 0; int mapTotal = 768; for (int i = 0; i <= n; i++) { iterRound = i; String input = prev_output; String output = dataDir + "/output" + "/iteration" + i; Job job = getJob(input, output, dataDir, i); if (i == n) { numReduceTasks = 1; } //job.setNumMapTasks(200); if (numOutput > mapTotal) { FileInputFormat.setMaxInputSplitSize(job, 10 * (8 * n + 10) + numOutput * (8 * n + 10) / 3000); FileInputFormat.setMinInputSplitSize(job, Math.max((8 * n + 10), numOutput * (8 * n + 10) / 5000)); } else { FileInputFormat.setMaxInputSplitSize(job, (8 * n + 10)); } /* if( i == 0 ) { job.setNumReduceTasks(1); } else { job.setNumReduceTasks(0); } */ job.setNumReduceTasks(0); job.waitForCompletion(true); // if failed ???? prev_output = output; Counters counters = job.getCounters(); Counter counter = counters.findCounter("MyCounter", "Map Output Counter"); numOutput = counter.getValue(); totalNodes += numOutput; cutbyDEE += counters.findCounter("MyCounter", "Cut By DEE").getValue(); searchedNodes += totalNodes + cutbyDEE + counters.findCounter("MyCounter", "Cut By Bound").getValue(); System.out.println(numOutput + " " + (8 * n + 10) + " " + (numOutput * (8 * n + 10) / 768)); } System.out.println("searchedNodes " + searchedNodes); System.out.println(totalNodes); System.out.println("cut by dee " + cutbyDEE); }
From source file:bdss.cmu.edu.Sort.java
License:Apache License
public static void main(String[] args) throws Exception { int res = ToolRunner.run(new Configuration(), new Sort(), args); System.exit(res);//from w w w . j a v a 2 s. c o m }
From source file:be.uantwerpen.adrem.bigfim.BigFIMDriver.java
License:Apache License
private boolean canStartPrefixGeneration(FIMOptions opt, int phase) throws IOException { Path path = new Path(opt.outputDir + separator + "tg" + phase); return phase >= opt.prefixLength && path.getFileSystem(new Configuration()).exists(path); }
From source file:be.uantwerpen.adrem.bigfim.BigFIMDriver.java
License:Apache License
private boolean canStartMining(FIMOptions opt) throws IOException { Path path = new Path(opt.outputDir + separator + "pg"); return path.getFileSystem(new Configuration()).exists(path); }
From source file:be.uantwerpen.adrem.bigfim.BigFIMDriver.java
License:Apache License
protected int startAprioriPhase(FIMOptions opt) throws IOException, InterruptedException, ClassNotFoundException, URISyntaxException { long nrLines = -1; int i = 1;//from www.j a v a 2s.c o m boolean run = true; while (run) { if (i == 1) { String outputDir = opt.outputDir + separator + "ap" + i; String info = "Apriori Phase " + i; run = runAprioriOncPhaseOnce(opt, nrLines, i, info, outputDir, null); } else { Path path = new Path(opt.outputDir + separator + "tg" + (i - 1)); for (FileStatus status : path.getFileSystem(new Configuration()).listStatus(path)) { String cacheFile = status.getPath().toString(); String trieGroupCount = cacheFile.substring(cacheFile.lastIndexOf('/'), cacheFile.length()); trieGroupCount = trieGroupCount.split("-")[1]; String outputDir = opt.outputDir + separator + "ap" + i + "-trieGroup" + trieGroupCount; System.out.println("CacheFile " + cacheFile); String info = "Apriori Phase " + i + ", Trie Group " + trieGroupCount; run = runAprioriOncPhaseOnce(opt, nrLines, i, info, outputDir, cacheFile); } } i++; } return i - 1; }