List of usage examples for org.apache.hadoop.conf Configuration Configuration
public Configuration()
From source file:bigfat.hadoop.HDFSDirInputStream.java
License:Apache License
public HDFSDirInputStream(String dir) throws IOException { this(FileSystem.get(new Configuration()), dir, null); }
From source file:bigfat.hadoop.HDFSDirInputStream.java
License:Apache License
/** * Test case, input int dir wants to read and the file to output * // w w w. ja va 2 s. co m * @param args * @throws IOException */ public static void main(String args[]) throws IOException { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); HDFSDirInputStream inp = new HDFSDirInputStream(fs, args[0]); FileOutputStream ops = new FileOutputStream(args[1]); int r; while ((r = inp.read()) != -1) { ops.write(r); } ops.close(); }
From source file:bigimp.BuildForest.java
License:Apache License
public static void main(String[] args) throws Exception { ToolRunner.run(new Configuration(), new BuildForest(), args); }
From source file:bigsatgps.BigDataHandler.java
License:Open Source License
/** * * @param infile/* w ww.ja v a 2s . co m*/ * @return * @throws Exception */ public String ImageToSequence(String infile) throws Exception { String log4jConfPath = "lib/log4j.properties"; PropertyConfigurator.configure(log4jConfPath); confHadoop = new Configuration(); confHadoop.addResource(new Path("/hadoop/projects/hadoop-1.0.4/conf/core-site.xml")); confHadoop.addResource(new Path("/hadoop/projects/hadoop-1.0.4/conf/hdfs-site.xml")); FileSystem fs = FileSystem.get(confHadoop); Path inPath = new Path(infile); String outfile = infile.substring(0, infile.indexOf(".")) + ".seq"; Path outPath = new Path(outfile); System.out.println(); System.out.println("Successfully created the sequencefile " + outfile); FSDataInputStream in = null; Text key = new Text(); BytesWritable value = new BytesWritable(); SequenceFile.Writer writer = null; try { in = fs.open(inPath); byte buffer[] = new byte[in.available()]; in.read(buffer); writer = SequenceFile.createWriter(fs, confHadoop, outPath, key.getClass(), value.getClass()); writer.append(new Text(inPath.getName()), new BytesWritable(buffer)); IOUtils.closeStream(writer); return outfile; } catch (IOException e) { System.err.println("Exception MESSAGES = " + e.getMessage()); IOUtils.closeStream(writer); return null; } }
From source file:binningbycategories.BinningbyCategories.java
/** * @param args the command line arguments * @throws java.lang.Exception// ww w . ja v a 2 s . c o m */ public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Job job = new Job(conf, "Binning"); job.setJarByClass(BinningbyCategories.class); job.setMapperClass(YouTubeBinMapper.class); job.setNumReduceTasks(0); TextInputFormat.setInputPaths(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); // Configure the MultipleOutputs by adding an output called "bins" // With the proper output format and mapper key/value pairs MultipleOutputs.addNamedOutput(job, "bins", TextOutputFormat.class, Text.class, NullWritable.class); // Enable the counters for the job // If there is a significant number of different named outputs, this // should be disabled MultipleOutputs.setCountersEnabled(job, true); System.exit(job.waitForCompletion(true) ? 0 : 2); }
From source file:BinningByState.Driver.java
public static void main(String args[]) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "BinningByState"); MultipleOutputs.addNamedOutput(job, "bins", TextOutputFormat.class, Text.class, NullWritable.class); MultipleOutputs.setCountersEnabled(job, true); job.setJarByClass(Driver.class); job.setMapperClass(BinningMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(NullWritable.class); job.setNumReduceTasks(0);/*w w w . j a v a 2 s .co m*/ // job.setOutputKeyClass(Text.class); // job.setOutputValueClass(NullWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:bixo.hadoop.HadoopConfigured.java
License:Apache License
public HadoopConfigured() { _conf = new Configuration(); }
From source file:biz.hangyang.knnspark.spark.KNNClassifySpark.java
public static JavaPairRDD<Entity, Object> calKDistance(final String trainingDataPath, String testingDataPath, final int k, final Map<Object, Double> weightMap, JavaSparkContext sc, int partition, final Accumulator<Integer> accum) { JavaRDD<String> testingDataRDD = sc.textFile(testingDataPath, partition); //?Entity/*from www . j a va 2 s. c om*/ JavaRDD<Entity> testingEntityRDD = testingDataRDD.map(new Function<String, Entity>() { @Override public Entity call(String line) throws Exception { return new GeneEntity(line); } }); //??????K??KV JavaPairRDD<Entity, KDistance> ekRDD = testingEntityRDD .mapPartitionsToPair(new PairFlatMapFunction<Iterator<Entity>, Entity, KDistance>() { @Override public Iterable<Tuple2<Entity, KDistance>> call(Iterator<Entity> t) throws Exception { //?PARTITION? List<Entity> entityList = new ArrayList<>(); while (t.hasNext()) { entityList.add(t.next()); } //??LIST List<KDistance> kDistanceList = new ArrayList<>(); for (int i = 0; i < entityList.size(); i++) { kDistanceList.add(new KDistance(k)); } //???hdfs Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(trainingDataPath), conf); FSDataInputStream in = fs.open(new Path(trainingDataPath)); BufferedReader br = new BufferedReader(new InputStreamReader(in, "UTF-8")); String line; while ((line = br.readLine()) != null) { Entity lineEntity = new GeneEntity(line); for (int i = 0; i < entityList.size(); i++) { kDistanceList.get(i).add(new DemoDistanceCatagory( lineEntity.distance(entityList.get(i)), lineEntity.category)); } } List<Tuple2<Entity, KDistance>> tList = new ArrayList<>(); for (int i = 0; i < entityList.size(); i++) { tList.add(new Tuple2<>(entityList.get(i), kDistanceList.get(i))); } return tList; } }); JavaPairRDD<Entity, Object> eoRDD = ekRDD .mapToPair(new PairFunction<Tuple2<Entity, KDistance>, Entity, Object>() { @Override public Tuple2<Entity, Object> call(Tuple2<Entity, KDistance> t) throws Exception { KDistance kDistance = t._2(); //??? Object catagory = KDistance.getCatagory(kDistance.get(), weightMap); if (t._1().category.equals(catagory)) { accum.add(1); } return new Tuple2<>(t._1(), catagory); } }); return eoRDD; }
From source file:bme.iclef.hadoop.file2seq.LocalSetup.java
License:Apache License
/** Sets up Configuration and LocalFileSystem instances for * Hadoop. Throws Exception if they fail. Does not load any * Hadoop XML configuration files, just sets the minimum * configuration necessary to use the local file system. *//*from w ww .j ava2s.c o m*/ public LocalSetup() throws Exception { config = new Configuration(); /* Normally set in hadoop-default.xml, without it you get * "java.io.IOException: No FileSystem for scheme: file" */ config.set("fs.file.impl", "org.apache.hadoop.fs.LocalFileSystem"); fileSystem = FileSystem.get(config); if (fileSystem.getConf() == null) { /* This happens if the FileSystem is not properly * initialized, causes NullPointerException later. */ throw new Exception("LocalFileSystem configuration is null"); } }
From source file:boa.compiler.Test.java
License:Apache License
public static void main(String[] args) throws IOException, URISyntaxException { Configuration configuration = new Configuration(); FileSystem hdfs = FileSystem.get(new URI("hdfs://localhost:54310"), configuration); FileStatus[] fileStatus = hdfs.listStatus(new Path("hdfs://localhost:54310/ast/")); Path[] paths = FileUtil.stat2Paths(fileStatus); System.out.println("***** Contents of the Directory *****"); for (Path path : paths) { System.out.println(path); }/*from www . j av a2s. c o m*/ }