List of usage examples for org.apache.hadoop.fs FileSystem getFileStatus
public abstract FileStatus getFileStatus(Path f) throws IOException;
From source file:edu.umn.cs.spatialHadoop.operations.Head.java
License:Open Source License
/** * Reads a maximum of n lines from the given file. * @param fs//from w ww. j a v a2 s .co m * @param p * @param n * @return * @throws IOException */ public static String[] head(FileSystem fs, Path p, int n) throws IOException { String[] lines = new String[n]; FileStatus fstatus = fs.getFileStatus(p); TaskAttemptContext context = createDummyContext(); LineRecordReader lineReader = new LineRecordReader(); FileSplit split; if (p.getName().endsWith(".rtree")) { // R-tree indexed file FSDataInputStream in = fs.open(p); in.skip(8); // Skip R-tree signature int treeHeaderSize = RTree.skipHeader(in); in.close(); split = new FileSplit(p, treeHeaderSize + 8, fstatus.getLen() - treeHeaderSize - 8, new String[0]); } else { split = new FileSplit(p, 0, fstatus.getLen(), new String[0]); } lineReader.initialize(split, context); int numOfLines = 0; for (numOfLines = 0; numOfLines < lines.length && lineReader.nextKeyValue(); numOfLines++) { lines[numOfLines] = lineReader.getCurrentValue().toString(); } lineReader.close(); return lines; }
From source file:edu.umn.cs.spatialHadoop.operations.Indexer.java
License:Open Source License
private static void indexLocal(Path inPath, Path outPath, OperationsParams params) throws IOException { JobConf job = new JobConf(params); String sindex = params.get("sindex"); Partitioner partitioner = createPartitioner(inPath, outPath, job, sindex); // Start reading input file Vector<InputSplit> splits = new Vector<InputSplit>(); final ShapeIterInputFormat inputFormat = new ShapeIterInputFormat(); FileSystem inFs = inPath.getFileSystem(params); FileStatus inFStatus = inFs.getFileStatus(inPath); if (inFStatus != null && !inFStatus.isDir()) { // One file, retrieve it immediately. // This is useful if the input is a hidden file which is automatically // skipped by FileInputFormat. We need to plot a hidden file for the case // of plotting partition boundaries of a spatial index splits.add(new FileSplit(inPath, 0, inFStatus.getLen(), new String[0])); } else {/*from w ww . j a v a2 s . co m*/ ShapeIterInputFormat.addInputPath(job, inPath); for (InputSplit s : inputFormat.getSplits(job, 1)) splits.add(s); } // Copy splits to a final array to be used in parallel final FileSplit[] fsplits = splits.toArray(new FileSplit[splits.size()]); boolean replicate = job.getBoolean("replicate", false); final IndexRecordWriter<Shape> recordWriter = new IndexRecordWriter<Shape>(partitioner, replicate, sindex, outPath, params); for (FileSplit fsplit : fsplits) { RecordReader<Rectangle, Iterable<? extends Shape>> reader = inputFormat.getRecordReader(fsplit, job, null); Rectangle partitionMBR = reader.createKey(); Iterable<? extends Shape> shapes = reader.createValue(); final IntWritable partitionID = new IntWritable(); while (reader.next(partitionMBR, shapes)) { if (replicate) { // Replicate each shape to all overlapping partitions for (final Shape s : shapes) { partitioner.overlapPartitions(s, new ResultCollector<Integer>() { @Override public void collect(Integer id) { partitionID.set(id); try { recordWriter.write(partitionID, s); } catch (IOException e) { throw new RuntimeException(e); } } }); } } else { for (Shape s : shapes) { partitionID.set(partitioner.overlapPartition(s)); recordWriter.write(partitionID, s); } } } reader.close(); } recordWriter.close(null); }
From source file:edu.umn.cs.spatialHadoop.operations.Intersects.java
License:Open Source License
public static <S extends Shape> long intersects(Path[] inFiles, Path userOutputPath, OperationsParams params) throws IOException, InterruptedException { JobConf job = new JobConf(params, Intersects.class); LOG.info("Intersects journey starts ...."); FileSystem inFs = inFiles[0].getFileSystem(job); Path outputPath = userOutputPath; if (outputPath == null) { FileSystem outFs = FileSystem.get(job); do {/*from w w w .j a va2 s. c o m*/ outputPath = new Path(inFiles[0].getName() + ".sjmr_" + (int) (Math.random() * 1000000)); } while (outFs.exists(outputPath)); } FileSystem outFs = outputPath.getFileSystem(job); ClusterStatus clusterStatus = new JobClient(job).getClusterStatus(); job.setJobName("Intersects"); job.setMapperClass(IntersectsMap.class); job.setMapOutputKeyClass(IntWritable.class); job.setMapOutputValueClass(IndexedText.class); job.setNumMapTasks(5 * Math.max(1, clusterStatus.getMaxMapTasks())); job.setLong("mapred.min.split.size", Math.max(inFs.getFileStatus(inFiles[0]).getBlockSize(), inFs.getFileStatus(inFiles[1]).getBlockSize())); job.setReducerClass(IntersectsReduce.class); job.setNumReduceTasks(Math.max(1, clusterStatus.getMaxReduceTasks())); job.setInputFormat(ShapeLineInputFormat.class); if (job.getBoolean("output", true)) job.setOutputFormat(TextOutputFormat.class); else job.setOutputFormat(NullOutputFormat.class); ShapeLineInputFormat.setInputPaths(job, inFiles); // Calculate and set the dimensions of the grid to use in the map phase long total_size = 0; Rectangle mbr = new Rectangle(Double.MAX_VALUE, Double.MAX_VALUE, -Double.MAX_VALUE, -Double.MAX_VALUE); for (Path file : inFiles) { FileSystem fs = file.getFileSystem(params); Rectangle file_mbr = FileMBR.fileMBR(file, params); mbr.expand(file_mbr); total_size += FileUtil.getPathSize(fs, file); } // If the largest file is globally indexed, use its partitions total_size += total_size * job.getFloat(SpatialSite.INDEXING_OVERHEAD, 0.2f); int sjmrPartitioningGridFactor = params.getInt(PartitioiningFactor, 20); int num_cells = (int) Math.max(1, total_size * sjmrPartitioningGridFactor / outFs.getDefaultBlockSize(outputPath)); LOG.info("Number of cells is configured to be " + num_cells); OperationsParams.setInactiveModeFlag(job, InactiveMode, isReduceInactive); OperationsParams.setJoiningThresholdPerOnce(job, JoiningThresholdPerOnce, joiningThresholdPerOnce); OperationsParams.setFilterOnlyModeFlag(job, isFilterOnlyMode, isFilterOnly); GridInfo gridInfo = new GridInfo(mbr.x1, mbr.y1, mbr.x2, mbr.y2); gridInfo.calculateCellDimensions(num_cells); OperationsParams.setShape(job, PartitionGrid, gridInfo); TextOutputFormat.setOutputPath(job, outputPath); if (OperationsParams.isLocal(job, inFiles)) { // Enforce local execution if explicitly set by user or for small files job.set("mapred.job.tracker", "local"); } // Start the job RunningJob runningJob = JobClient.runJob(job); Counters counters = runningJob.getCounters(); Counter outputRecordCounter = counters.findCounter(Task.Counter.REDUCE_OUTPUT_RECORDS); final long resultCount = outputRecordCounter.getValue(); return resultCount; }
From source file:edu.umn.cs.spatialHadoop.operations.KNN.java
License:Open Source License
private static <S extends Shape> long knnLocal(Path inFile, Path outPath, OperationsParams params) throws IOException, InterruptedException { int iterations = 0; FileSystem fs = inFile.getFileSystem(params); Point queryPoint = (Point) OperationsParams.getShape(params, "point"); int k = params.getInt("k", 1); // Top-k objects are retained in this object PriorityQueue<ShapeWithDistance<S>> knn = new KNNObjects<ShapeWithDistance<S>>(k); SpatialInputFormat3<Rectangle, Shape> inputFormat = new SpatialInputFormat3<Rectangle, Shape>(); final GlobalIndex<Partition> gIndex = SpatialSite.getGlobalIndex(fs, inFile); double kthDistance = Double.MAX_VALUE; if (gIndex != null) { // There is a global index, use it PriorityQueue<ShapeWithDistance<Partition>> partitionsToProcess = new PriorityQueue<KNN.ShapeWithDistance<Partition>>() { {//ww w. ja v a 2s. c o m initialize(gIndex.size()); } @Override protected boolean lessThan(Object a, Object b) { return ((ShapeWithDistance<Partition>) a).distance < ((ShapeWithDistance<Partition>) b).distance; } }; for (Partition p : gIndex) { double distance = p.getMinDistanceTo(queryPoint.x, queryPoint.y); partitionsToProcess.insert(new ShapeWithDistance<Partition>(p.clone(), distance)); } while (partitionsToProcess.size() > 0 && partitionsToProcess.top().distance <= kthDistance) { ShapeWithDistance<Partition> partitionToProcess = partitionsToProcess.pop(); // Process this partition Path partitionPath = new Path(inFile, partitionToProcess.shape.filename); long length = fs.getFileStatus(partitionPath).getLen(); FileSplit fsplit = new FileSplit(partitionPath, 0, length, new String[0]); RecordReader<Rectangle, Iterable<Shape>> reader = inputFormat.createRecordReader(fsplit, null); if (reader instanceof SpatialRecordReader3) { ((SpatialRecordReader3) reader).initialize(fsplit, params); } else if (reader instanceof RTreeRecordReader3) { ((RTreeRecordReader3) reader).initialize(fsplit, params); } else if (reader instanceof HDFRecordReader) { ((HDFRecordReader) reader).initialize(fsplit, params); } else { throw new RuntimeException("Unknown record reader"); } iterations++; while (reader.nextKeyValue()) { Iterable<Shape> shapes = reader.getCurrentValue(); for (Shape shape : shapes) { double distance = shape.distanceTo(queryPoint.x, queryPoint.y); if (distance <= kthDistance) knn.insert(new ShapeWithDistance<S>((S) shape.clone(), distance)); } } reader.close(); if (knn.size() >= k) kthDistance = knn.top().distance; } } else { // No global index, have to scan the whole file Job job = new Job(params); SpatialInputFormat3.addInputPath(job, inFile); List<InputSplit> splits = inputFormat.getSplits(job); for (InputSplit split : splits) { RecordReader<Rectangle, Iterable<Shape>> reader = inputFormat.createRecordReader(split, null); if (reader instanceof SpatialRecordReader3) { ((SpatialRecordReader3) reader).initialize(split, params); } else if (reader instanceof RTreeRecordReader3) { ((RTreeRecordReader3) reader).initialize(split, params); } else if (reader instanceof HDFRecordReader) { ((HDFRecordReader) reader).initialize(split, params); } else { throw new RuntimeException("Unknown record reader"); } iterations++; while (reader.nextKeyValue()) { Iterable<Shape> shapes = reader.getCurrentValue(); for (Shape shape : shapes) { double distance = shape.distanceTo(queryPoint.x, queryPoint.y); knn.insert(new ShapeWithDistance<S>((S) shape.clone(), distance)); } } reader.close(); } if (knn.size() >= k) kthDistance = knn.top().distance; } long resultCount = knn.size(); if (outPath != null && params.getBoolean("output", true)) { FileSystem outFS = outPath.getFileSystem(params); PrintStream ps = new PrintStream(outFS.create(outPath)); Vector<ShapeWithDistance<S>> resultsOrdered = new Vector<ShapeWithDistance<S>>((int) resultCount); resultsOrdered.setSize((int) resultCount); while (knn.size() > 0) { ShapeWithDistance<S> nextAnswer = knn.pop(); resultsOrdered.set(knn.size(), nextAnswer); } Text text = new Text(); for (ShapeWithDistance<S> answer : resultsOrdered) { text.clear(); TextSerializerHelper.serializeDouble(answer.distance, text, ','); answer.shape.toText(text); ps.println(text); } ps.close(); } TotalIterations.addAndGet(iterations); return resultCount; }
From source file:edu.umn.cs.spatialHadoop.operations.LocalSampler.java
License:Open Source License
/** * Read a random sample of up-to count from the input files. * @param files//from w w w . java2 s. c o m * @param ratioOrCount * @param output * @param conf * @return * @throws IOException * @throws InterruptedException */ public static long sampleLocal(Path[] files, float ratioOrCount, ResultCollector<Text> output, Configuration conf) throws IOException, InterruptedException { Vector<FileSplit> splits = new Vector<FileSplit>(); for (Path file : files) { FileSystem fs = file.getFileSystem(conf); if (fs.isFile(file)) { // A single file. Include it splits.add(new FileSplit(file, 0, fs.getFileStatus(file).getLen(), new String[0])); } else { // A directory. Include all contents FileStatus[] contents = fs.listStatus(file); for (FileStatus content : contents) { if (!content.isDirectory()) splits.add(new FileSplit(content.getPath(), 0, content.getLen(), new String[0])); } } } return sampleLocal(splits.toArray(new FileSplit[splits.size()]), ratioOrCount, output, conf); }
From source file:edu.umn.cs.spatialHadoop.operations.Overlaps.java
License:Open Source License
public static <S extends Shape> long overlaps(Path[] inFiles, Path userOutputPath, OperationsParams params) throws IOException, InterruptedException { JobConf job = new JobConf(params, Overlaps.class); LOG.info("Overlaps journey starts ...."); FileSystem inFs = inFiles[0].getFileSystem(job); Path outputPath = userOutputPath; if (outputPath == null) { FileSystem outFs = FileSystem.get(job); do {/*ww w .ja v a 2 s . c o m*/ outputPath = new Path(inFiles[0].getName() + ".sjmr_" + (int) (Math.random() * 1000000)); } while (outFs.exists(outputPath)); } FileSystem outFs = outputPath.getFileSystem(job); ClusterStatus clusterStatus = new JobClient(job).getClusterStatus(); job.setJobName("Overlaps"); job.setMapperClass(OverlapMap.class); job.setMapOutputKeyClass(IntWritable.class); job.setMapOutputValueClass(IndexedText.class); job.setNumMapTasks(5 * Math.max(1, clusterStatus.getMaxMapTasks())); job.setLong("mapred.min.split.size", Math.max(inFs.getFileStatus(inFiles[0]).getBlockSize(), inFs.getFileStatus(inFiles[1]).getBlockSize())); job.setReducerClass(OverlapReduce.class); job.setNumReduceTasks(Math.max(1, clusterStatus.getMaxReduceTasks())); job.setInputFormat(ShapeLineInputFormat.class); if (job.getBoolean("output", true)) job.setOutputFormat(TextOutputFormat.class); else job.setOutputFormat(NullOutputFormat.class); ShapeLineInputFormat.setInputPaths(job, inFiles); // Calculate and set the dimensions of the grid to use in the map phase long total_size = 0; Rectangle mbr = new Rectangle(Double.MAX_VALUE, Double.MAX_VALUE, -Double.MAX_VALUE, -Double.MAX_VALUE); for (Path file : inFiles) { FileSystem fs = file.getFileSystem(params); Rectangle file_mbr = FileMBR.fileMBR(file, params); mbr.expand(file_mbr); total_size += FileUtil.getPathSize(fs, file); } // If the largest file is globally indexed, use its partitions total_size += total_size * job.getFloat(SpatialSite.INDEXING_OVERHEAD, 0.2f); int sjmrPartitioningGridFactor = params.getInt(PartitioiningFactor, 20); int num_cells = (int) Math.max(1, total_size * sjmrPartitioningGridFactor / outFs.getDefaultBlockSize(outputPath)); LOG.info("Number of cells is configured to be " + num_cells); OperationsParams.setInactiveModeFlag(job, InactiveMode, isReduceInactive); OperationsParams.setJoiningThresholdPerOnce(job, JoiningThresholdPerOnce, joiningThresholdPerOnce); OperationsParams.setFilterOnlyModeFlag(job, isFilterOnlyMode, isFilterOnly); GridInfo gridInfo = new GridInfo(mbr.x1, mbr.y1, mbr.x2, mbr.y2); gridInfo.calculateCellDimensions(num_cells); OperationsParams.setShape(job, PartitionGrid, gridInfo); TextOutputFormat.setOutputPath(job, outputPath); if (OperationsParams.isLocal(job, inFiles)) { // Enforce local execution if explicitly set by user or for small files job.set("mapred.job.tracker", "local"); } // Start the job RunningJob runningJob = JobClient.runJob(job); Counters counters = runningJob.getCounters(); Counter outputRecordCounter = counters.findCounter(Task.Counter.REDUCE_OUTPUT_RECORDS); final long resultCount = outputRecordCounter.getValue(); return resultCount; }
From source file:edu.umn.cs.spatialHadoop.operations.Plot.java
License:Apache License
public static <S extends Shape> void plotMapReduce(Path inFile, Path outFile, Shape shape, int width, int height, Color color, boolean showBorders, boolean showBlockCount, boolean showRecordCount, boolean background) throws IOException { JobConf job = new JobConf(Plot.class); job.setJobName("Plot"); job.setMapperClass(PlotMap.class); ClusterStatus clusterStatus = new JobClient(job).getClusterStatus(); job.setNumMapTasks(clusterStatus.getMaxMapTasks() * 5); job.setReducerClass(PlotReduce.class); job.setNumReduceTasks(Math.max(1, clusterStatus.getMaxReduceTasks())); job.setMapOutputKeyClass(Rectangle.class); SpatialSite.setShapeClass(job, shape.getClass()); job.setMapOutputValueClass(shape.getClass()); FileSystem inFs = inFile.getFileSystem(job); Rectangle fileMbr = FileMBR.fileMBRMapReduce(inFs, inFile, shape, false); FileStatus inFileStatus = inFs.getFileStatus(inFile); CellInfo[] cellInfos;//from w w w . j a v a2s. c om GlobalIndex<Partition> gindex = SpatialSite.getGlobalIndex(inFs, inFile); if (gindex == null) { // A heap file. The map function should partition the file GridInfo gridInfo = new GridInfo(fileMbr.x1, fileMbr.y1, fileMbr.x2, fileMbr.y2); gridInfo.calculateCellDimensions(inFileStatus.getLen(), inFileStatus.getBlockSize()); cellInfos = gridInfo.getAllCells(); // Doesn't make sense to show any partition information in a heap file showBorders = showBlockCount = showRecordCount = false; } else { cellInfos = SpatialSite.cellsOf(inFs, inFile); } // Set cell information in the job configuration to be used by the mapper SpatialSite.setCells(job, cellInfos); // Adjust width and height to maintain aspect ratio if ((fileMbr.x2 - fileMbr.x1) / (fileMbr.y2 - fileMbr.y1) > (double) width / height) { // Fix width and change height height = (int) ((fileMbr.y2 - fileMbr.y1) * width / (fileMbr.x2 - fileMbr.x1)); } else { width = (int) ((fileMbr.x2 - fileMbr.x1) * height / (fileMbr.y2 - fileMbr.y1)); } LOG.info("Creating an image of size " + width + "x" + height); ImageOutputFormat.setFileMBR(job, fileMbr); ImageOutputFormat.setImageWidth(job, width); ImageOutputFormat.setImageHeight(job, height); job.setBoolean(ShowBorders, showBorders); job.setBoolean(ShowBlockCount, showBlockCount); job.setBoolean(ShowRecordCount, showRecordCount); job.setInt(StrokeColor, color.getRGB()); // Set input and output job.setInputFormat(ShapeInputFormat.class); ShapeInputFormat.addInputPath(job, inFile); // Set output committer which will stitch images together after all reducers // finish job.setOutputCommitter(PlotOutputCommitter.class); job.setOutputFormat(ImageOutputFormat.class); TextOutputFormat.setOutputPath(job, outFile); if (background) { JobClient jc = new JobClient(job); lastSubmittedJob = jc.submitJob(job); } else { lastSubmittedJob = JobClient.runJob(job); } }
From source file:edu.umn.cs.spatialHadoop.operations.Plot.java
License:Apache License
public static <S extends Shape> void plotLocal(Path inFile, Path outFile, S shape, int width, int height, Color color, boolean showBorders, boolean showBlockCount, boolean showRecordCount) throws IOException { FileSystem inFs = inFile.getFileSystem(new Configuration()); Rectangle fileMbr = FileMBR.fileMBRLocal(inFs, inFile, shape); LOG.info("FileMBR: " + fileMbr); // Adjust width and height to maintain aspect ratio if ((fileMbr.x2 - fileMbr.x1) / (fileMbr.y2 - fileMbr.y1) > (double) width / height) { // Fix width and change height height = (int) ((fileMbr.y2 - fileMbr.y1) * width / (fileMbr.x2 - fileMbr.x1)); } else {/* w w w . ja va 2 s. c o m*/ width = (int) ((fileMbr.x2 - fileMbr.x1) * height / (fileMbr.y2 - fileMbr.y1)); } double scale2 = (double) width * height / ((double) (fileMbr.x2 - fileMbr.x1) * (fileMbr.y2 - fileMbr.y1)); // Create an image BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB); Graphics2D graphics = image.createGraphics(); Color bg_color = new Color(0, 0, 0, 0); graphics.setBackground(bg_color); graphics.clearRect(0, 0, width, height); graphics.setColor(color); long fileLength = inFs.getFileStatus(inFile).getLen(); ShapeRecordReader<S> reader = new ShapeRecordReader<S>(inFs.open(inFile), 0, fileLength); Rectangle cell = reader.createKey(); while (reader.next(cell, shape)) { drawShape(graphics, shape, fileMbr, width, height, scale2); } reader.close(); graphics.dispose(); FileSystem outFs = outFile.getFileSystem(new Configuration()); OutputStream out = outFs.create(outFile, true); ImageIO.write(image, "png", out); out.close(); }
From source file:edu.umn.cs.spatialHadoop.operations.Plot.java
License:Apache License
/** * Combines images of different datasets into one image that is displayed * to users./* ww w . j a va 2 s. c o m*/ * @param fs The file system that contains the datasets and images * @param files Paths to directories which contains the datasets * @param includeBoundaries Also plot the indexing boundaries of datasets * @return An image that is the combination of all datasets images * @throws IOException */ public static BufferedImage combineImages(Configuration conf, Path[] files, boolean includeBoundaries, int width, int height) throws IOException { BufferedImage result = null; // Retrieve the MBRs of all datasets Rectangle allMbr = new Rectangle(Double.MAX_VALUE, Double.MAX_VALUE, -Double.MAX_VALUE, -Double.MAX_VALUE); for (Path file : files) { FileSystem fs = file.getFileSystem(conf); Rectangle mbr = FileMBR.fileMBR(fs, file, null); allMbr.expand(mbr); } // Adjust width and height to maintain aspect ratio if ((allMbr.x2 - allMbr.x1) / (allMbr.y2 - allMbr.y1) > (double) width / height) { // Fix width and change height height = (int) ((allMbr.y2 - allMbr.y1) * width / (allMbr.x2 - allMbr.x1)); } else { width = (int) ((allMbr.x2 - allMbr.x1) * height / (allMbr.y2 - allMbr.y1)); } result = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB); for (Path file : files) { FileSystem fs = file.getFileSystem(conf); if (fs.getFileStatus(file).isDir()) { // Retrieve the MBR of this dataset Rectangle mbr = FileMBR.fileMBR(fs, file, null); // Compute the coordinates of this image in the whole picture mbr.x1 = (mbr.x1 - allMbr.x1) * width / allMbr.getWidth(); mbr.x2 = (mbr.x2 - allMbr.x1) * width / allMbr.getWidth(); mbr.y1 = (mbr.y1 - allMbr.y1) * height / allMbr.getHeight(); mbr.y2 = (mbr.y2 - allMbr.y1) * height / allMbr.getHeight(); // Retrieve the image of this dataset Path imagePath = new Path(file, "_data.png"); if (!fs.exists(imagePath)) throw new RuntimeException("Image " + imagePath + " not ready"); FSDataInputStream imageFile = fs.open(imagePath); BufferedImage image = ImageIO.read(imageFile); imageFile.close(); // Draw the image Graphics graphics = result.getGraphics(); graphics.drawImage(image, (int) mbr.x1, (int) mbr.y1, (int) mbr.getWidth(), (int) mbr.getHeight(), null); graphics.dispose(); if (includeBoundaries) { // Plot also the image of the boundaries // Retrieve the image of the dataset boundaries imagePath = new Path(file, "_partitions.png"); if (fs.exists(imagePath)) { imageFile = fs.open(imagePath); image = ImageIO.read(imageFile); imageFile.close(); // Draw the image graphics = result.getGraphics(); graphics.drawImage(image, (int) mbr.x1, (int) mbr.y1, (int) mbr.getWidth(), (int) mbr.getHeight(), null); graphics.dispose(); } } } } return result; }
From source file:edu.umn.cs.spatialHadoop.operations.PyramidPlot.java
License:Apache License
private static void plotLocal(Path inFile, Path outFile, OperationsParams params) throws IOException { int tileWidth = params.getInt("tilewidth", 256); int tileHeight = params.getInt("tileheight", 256); Color strokeColor = params.getColor("color", Color.BLACK); String hdfDataset = (String) params.get("dataset"); Shape shape = hdfDataset != null ? new NASARectangle() : (Shape) params.getShape("shape", null); Shape plotRange = params.getShape("rect", null); String valueRangeStr = (String) params.get("valuerange"); MinMax valueRange;//w w w . j a v a 2s .c o m if (valueRangeStr == null) { valueRange = null; } else { String[] parts = valueRangeStr.split(","); valueRange = new MinMax(Integer.parseInt(parts[0]), Integer.parseInt(parts[1])); } InputSplit[] splits; FileSystem inFs = inFile.getFileSystem(params); FileStatus inFStatus = inFs.getFileStatus(inFile); if (inFStatus != null && !inFStatus.isDir()) { // One file, retrieve it immediately. // This is useful if the input is a hidden file which is automatically // skipped by FileInputFormat. We need to plot a hidden file for the case // of plotting partition boundaries of a spatial index splits = new InputSplit[] { new FileSplit(inFile, 0, inFStatus.getLen(), new String[0]) }; } else { JobConf job = new JobConf(params); ShapeInputFormat<Shape> inputFormat = new ShapeInputFormat<Shape>(); ShapeInputFormat.addInputPath(job, inFile); splits = inputFormat.getSplits(job, 1); } boolean vflip = params.is("vflip"); Rectangle fileMBR; if (plotRange != null) { fileMBR = plotRange.getMBR(); } else if (hdfDataset != null) { // Plotting a NASA file fileMBR = new Rectangle(-180, -90, 180, 90); } else { fileMBR = FileMBR.fileMBR(inFile, params); } boolean keepAspectRatio = params.is("keep-ratio", true); if (keepAspectRatio) { // Adjust width and height to maintain aspect ratio if (fileMBR.getWidth() > fileMBR.getHeight()) { fileMBR.y1 -= (fileMBR.getWidth() - fileMBR.getHeight()) / 2; fileMBR.y2 = fileMBR.y1 + fileMBR.getWidth(); } else { fileMBR.x1 -= (fileMBR.getHeight() - fileMBR.getWidth() / 2); fileMBR.x2 = fileMBR.x1 + fileMBR.getHeight(); } } if (hdfDataset != null) { // Collects some stats about the HDF file if (valueRange == null) valueRange = Aggregate.aggregate(new Path[] { inFile }, params); NASAPoint.minValue = valueRange.minValue; NASAPoint.maxValue = valueRange.maxValue; NASAPoint.setColor1(params.getColor("color1", Color.BLUE)); NASAPoint.setColor2(params.getColor("color2", Color.RED)); NASAPoint.gradientType = params.getGradientType("gradient", NASAPoint.GradientType.GT_HUE); } boolean adaptiveSampling = params.getBoolean("sample", false); int numLevels = params.getInt("numlevels", 7); float[] levelProb = new float[numLevels]; double[] scale2 = new double[numLevels]; double[] scale = new double[numLevels]; levelProb[0] = params.getFloat(GeometricPlot.AdaptiveSampleRatio, 0.1f); // Size of the whole file in pixels at the f scale2[0] = (double) tileWidth * tileHeight / (fileMBR.getWidth() * fileMBR.getHeight()); scale[0] = Math.sqrt(scale2[0]); for (int level = 1; level < numLevels; level++) { levelProb[level] = levelProb[level - 1] * 4; scale2[level] = scale2[level - 1] * (1 << level) * (1 << level); scale[level] = scale[level - 1] * (1 << level); } Map<TileIndex, BufferedImage> tileImages = new HashMap<PyramidPlot.TileIndex, BufferedImage>(); Map<TileIndex, Graphics2D> tileGraphics = new HashMap<PyramidPlot.TileIndex, Graphics2D>(); GridInfo bottomGrid = new GridInfo(fileMBR.x1, fileMBR.y1, fileMBR.x2, fileMBR.y2); bottomGrid.rows = bottomGrid.columns = (int) Math.round(Math.pow(2, numLevels - 1)); TileIndex tileIndex = new TileIndex(); boolean gradualFade = !(shape instanceof Point) && params.getBoolean("fade", false); for (InputSplit split : splits) { ShapeRecordReader<Shape> reader = new ShapeRecordReader<Shape>(params, (FileSplit) split); Rectangle cell = reader.createKey(); while (reader.next(cell, shape)) { Rectangle shapeMBR = shape.getMBR(); if (shapeMBR != null) { int min_level = 0; if (adaptiveSampling) { // Special handling for NASA data double p = Math.random(); // Skip levels that do not satisfy the probability while (min_level < numLevels && p > levelProb[min_level]) min_level++; } java.awt.Rectangle overlappingCells = bottomGrid.getOverlappingCells(shapeMBR); for (tileIndex.level = numLevels - 1; tileIndex.level >= min_level; tileIndex.level--) { if (gradualFade && !(shape instanceof Point)) { double areaInPixels = (shapeMBR.getWidth() + shapeMBR.getHeight()) * scale[tileIndex.level]; if (areaInPixels < 1.0 && Math.round(areaInPixels * 255) < 1.0) { // This shape can be safely skipped as it is too small to be plotted return; } } for (int i = 0; i < overlappingCells.width; i++) { tileIndex.x = i + overlappingCells.x; for (int j = 0; j < overlappingCells.height; j++) { tileIndex.y = j + overlappingCells.y; // Draw in image associated with this tile Graphics2D g; { g = tileGraphics.get(tileIndex); if (g == null) { TileIndex key = tileIndex.clone(); BufferedImage image = new BufferedImage(tileWidth, tileHeight, BufferedImage.TYPE_INT_ARGB); if (tileImages.put(key, image) != null) throw new RuntimeException( "Error! Image is already there but graphics is not " + tileIndex); Color bg_color = new Color(0, 0, 0, 0); try { g = image.createGraphics(); } catch (Throwable e) { g = new SimpleGraphics(image); } g.setBackground(bg_color); g.clearRect(0, 0, tileWidth, tileHeight); g.setColor(strokeColor); // Coordinates of this tile in image coordinates g.translate(-(tileWidth * tileIndex.x), -(tileHeight * tileIndex.y)); tileGraphics.put(key, g); } } shape.draw(g, fileMBR, tileWidth * (1 << tileIndex.level), tileHeight * (1 << tileIndex.level), scale2[tileIndex.level]); } } // Shrink overlapping cells to match the upper level int updatedX1 = overlappingCells.x / 2; int updatedY1 = overlappingCells.y / 2; int updatedX2 = (overlappingCells.x + overlappingCells.width - 1) / 2; int updatedY2 = (overlappingCells.y + overlappingCells.height - 1) / 2; overlappingCells.x = updatedX1; overlappingCells.y = updatedY1; overlappingCells.width = updatedX2 - updatedX1 + 1; overlappingCells.height = updatedY2 - updatedY1 + 1; } } } reader.close(); } // Write image to output for (Map.Entry<TileIndex, Graphics2D> tileGraph : tileGraphics.entrySet()) { tileGraph.getValue().dispose(); } FileSystem outFS = outFile.getFileSystem(params); for (Map.Entry<TileIndex, BufferedImage> tileImage : tileImages.entrySet()) { tileIndex = tileImage.getKey(); BufferedImage image = tileImage.getValue(); if (vflip) { AffineTransform tx = AffineTransform.getScaleInstance(1, -1); tx.translate(0, -image.getHeight()); AffineTransformOp op = new AffineTransformOp(tx, AffineTransformOp.TYPE_NEAREST_NEIGHBOR); image = op.filter(image, null); tileIndex.y = ((1 << tileIndex.level) - 1) - tileIndex.y; } Path imagePath = new Path(outFile, tileIndex.getImageFileName()); FSDataOutputStream outStream = outFS.create(imagePath); ImageIO.write(image, "png", outStream); outStream.close(); } }