List of usage examples for org.apache.hadoop.fs FileSystem create
public FSDataOutputStream create(Path f) throws IOException
From source file:com.ricemap.spateDB.util.RandomSpatialGenerator.java
License:Apache License
public static void generateMapReduce(Path file, Prism mbr, long size, long blocksize, Shape shape, String sindex, long seed, int rectsize, RandomShapeGenerator.DistributionType type, boolean overwrite) throws IOException { JobConf job = new JobConf(RandomSpatialGenerator.class); job.setJobName("Generator"); FileSystem outFs = file.getFileSystem(job); // Overwrite output file if (outFs.exists(file)) { if (overwrite) outFs.delete(file, true);/*from w ww .j a v a2 s. c o m*/ else throw new RuntimeException( "Output file '" + file + "' already exists and overwrite flag is not set"); } // Set generation parameters in job job.setLong(RandomShapeGenerator.GenerationSize, size); SpatialSite.setPrism(job, RandomShapeGenerator.GenerationMBR, mbr); if (seed != 0) job.setLong(RandomShapeGenerator.GenerationSeed, seed); if (rectsize != 0) job.setInt(RandomShapeGenerator.GenerationRectSize, rectsize); if (type != null) job.set(RandomShapeGenerator.GenerationType, type.toString()); ClusterStatus clusterStatus = new JobClient(job).getClusterStatus(); // Set input format and map class job.setInputFormat(RandomInputFormat.class); job.setMapperClass(Repartition.RepartitionMap.class); job.setMapOutputKeyClass(IntWritable.class); job.setMapOutputValueClass(shape.getClass()); job.setNumMapTasks(10 * Math.max(1, clusterStatus.getMaxMapTasks())); SpatialSite.setShapeClass(job, shape.getClass()); if (blocksize != 0) { job.setLong(SpatialSite.LOCAL_INDEX_BLOCK_SIZE, blocksize); } CellInfo[] cells; if (sindex == null) { cells = new CellInfo[] { new CellInfo(1, mbr) }; } else if (sindex.equals("grid")) { GridInfo gridInfo = new GridInfo(mbr.t1, mbr.x1, mbr.y1, mbr.t2, mbr.x2, mbr.y2); FileSystem fs = file.getFileSystem(job); if (blocksize == 0) { blocksize = fs.getDefaultBlockSize(file); } int numOfCells = Repartition.calculateNumberOfPartitions(job, size, fs, file, blocksize); gridInfo.calculateCellDimensions(numOfCells); cells = gridInfo.getAllCells(); } else { throw new RuntimeException("Unsupported spatial index: " + sindex); } SpatialSite.setCells(job, cells); // Do not set a reduce function. Use the default identity reduce function if (cells.length == 1) { // All objects are in one partition. No need for a reduce phase job.setNumReduceTasks(0); } else { // More than one partition. Need a reduce phase to group shapes of the // same partition together job.setReducerClass(RepartitionReduce.class); job.setNumReduceTasks( Math.max(1, Math.min(cells.length, (clusterStatus.getMaxReduceTasks() * 9 + 5) / 10))); } // Set output path FileOutputFormat.setOutputPath(job, file); if (sindex == null || sindex.equals("grid")) { job.setOutputFormat(GridOutputFormat.class); } else { throw new RuntimeException("Unsupported spatial index: " + sindex); } JobClient.runJob(job); // Concatenate all master files into one file FileStatus[] resultFiles = outFs.listStatus(file, new PathFilter() { @Override public boolean accept(Path path) { return path.getName().contains("_master"); } }); String ext = resultFiles[0].getPath().getName() .substring(resultFiles[0].getPath().getName().lastIndexOf('.')); Path masterPath = new Path(file, "_master" + ext); OutputStream destOut = outFs.create(masterPath); byte[] buffer = new byte[4096]; for (FileStatus f : resultFiles) { InputStream in = outFs.open(f.getPath()); int bytes_read; do { bytes_read = in.read(buffer); if (bytes_read > 0) destOut.write(buffer, 0, bytes_read); } while (bytes_read > 0); in.close(); outFs.delete(f.getPath(), false); } destOut.close(); // Plot an image for the partitions used in file Path imagePath = new Path(file, "_partitions.png"); int imageSize = (int) (Math.sqrt(cells.length) * 300); Plot.plotLocal(masterPath, imagePath, new Partition(), imageSize, imageSize, Color.BLACK, false, false, false); }
From source file:com.rim.logdriver.util.IndexLogs.java
License:Apache License
public static void main(String args[]) throws IOException, ParseException { // Create blank map for components Map<String, Map<String, Map<String, Map<String, Component>>>> data = new HashMap<String, Map<String, Map<String, Map<String, Component>>>>(); // Set the output format Boolean humanReadable = false; Boolean writeIndex = true;/* w w w . ja v a2s . c o m*/ for (int i = 0; i < args.length; i++) { if (args[i].matches("-t")) { humanReadable = true; } else if (args[i].matches("-n")) { writeIndex = false; } else { System.out.println( "Usage: indexlogs [-t -n]\n -t Print results to STDOUT in human-readable tree\n" + " -n Don't write index files into HDFS\n"); System.exit(0); } } // Set up HDFS filesystem Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); // Search the /service folder for matching paths if (!humanReadable && !writeIndex) { System.out.println("Warning: -n set without -t, not doing anything.\n"); System.exit(0); } System.out.println("Indexing logs..."); findComponents(data, fs, new Path("/service")); // Output the generated index if (humanReadable) { humanPrint(data); System.out.println(""); } if (writeIndex) { long currentTime = System.currentTimeMillis() / 1000; FSDataOutputStream outputCSV = fs.create(new Path("/service/_index/logindex." + currentTime + ".csv")); writeCSV(data, outputCSV); outputCSV.close(); System.out.println("Index files written to /service/_index/logindex." + currentTime + ".csv and /service/_index/logindex." + currentTime + ".json"); FSDataOutputStream outputJSON = fs .create(new Path("/service/_index/logindex." + currentTime + ".json")); writeJSON(data, outputJSON); outputJSON.close(); } }
From source file:com.ripariandata.timberwolf.writer.hive.HiveMailWriterTest.java
License:Apache License
@Test @SuppressWarnings("deprecation") public void testWritingToNewTable() throws SQLException, IOException { Connection hive = mock(Connection.class); FileSystem hdfs = mock(FileSystem.class); PreparedStatement showStmt = createMockTableExistsResponse(hive, false); Statement createStmt = mock(Statement.class); when(hive.createStatement()).thenReturn(createStmt); PreparedStatement loadStmt = mock(PreparedStatement.class); when(hive.prepareStatement("load data inpath ? into table new_table")).thenReturn(loadStmt); when(hdfs.exists(eq(new Path("/tmp/timberwolf")))).thenReturn(false); when(hdfs.create(any(Path.class))).thenReturn(new FSDataOutputStream(new ByteArrayOutputStream())); HiveMailWriter writer = new HiveMailWriter(hdfs, hive, "new_table"); writer.write(new ArrayList<MailboxItem>()); verify(showStmt).setString(1, "new_table"); String s = "create table new_table \\([^)]+\\)" + " row format delimited fields terminated by '\\\\037'" + " stored as sequencefile"; verify(createStmt).executeQuery(matches(s)); verify(loadStmt).setString(eq(1), startsWith("/tmp/timberwolf/")); verify(loadStmt).executeQuery();//from w w w. j a v a 2 s. c o m verify(hdfs).mkdirs(eq(new Path("/tmp/timberwolf"))); verify(hdfs).delete(any(Path.class), eq(false)); }
From source file:com.ripariandata.timberwolf.writer.hive.HiveMailWriterTest.java
License:Apache License
@Test @SuppressWarnings("deprecation") public void testWritingToExistingTable() throws SQLException, IOException { Connection hive = mock(Connection.class); FileSystem hdfs = mock(FileSystem.class); PreparedStatement showStmt = createMockTableExistsResponse(hive, true); PreparedStatement loadStmt = mock(PreparedStatement.class); when(hive.prepareStatement("load data inpath ? into table new_table")).thenReturn(loadStmt); when(hdfs.exists(eq(new Path("/tmp/timberwolf")))).thenReturn(false); when(hdfs.create(any(Path.class))).thenReturn(new FSDataOutputStream(new ByteArrayOutputStream())); HiveMailWriter writer = new HiveMailWriter(hdfs, hive, "new_table"); writer.write(new ArrayList<MailboxItem>()); verify(showStmt).setString(1, "new_table"); verify(loadStmt).setString(eq(1), startsWith("/tmp/timberwolf/")); verify(loadStmt).executeQuery();/*from w w w.j a v a2 s . c o m*/ verify(hdfs).mkdirs(eq(new Path("/tmp/timberwolf"))); verify(hdfs).delete(any(Path.class), eq(false)); }
From source file:com.ripariandata.timberwolf.writer.hive.HiveMailWriterTest.java
License:Apache License
@Test @SuppressWarnings("deprecation") public void testWritingAndFailing() throws SQLException, IOException { Connection hive = mock(Connection.class); FileSystem hdfs = mock(FileSystem.class); PreparedStatement showStmt = createMockTableExistsResponse(hive, true); PreparedStatement loadStmt = mock(PreparedStatement.class); when(hive.prepareStatement("load data inpath ? into table new_table")).thenReturn(loadStmt); // Here is the change, we throw an exception instead of returning when(loadStmt.executeQuery()).thenThrow(new SQLException("mock failed to load")); when(hdfs.exists(eq(new Path("/tmp/timberwolf")))).thenReturn(true); when(hdfs.create(any(Path.class))).thenReturn(new FSDataOutputStream(new ByteArrayOutputStream())); HiveMailWriter writer = new HiveMailWriter(hdfs, hive, "new_table"); try {/*from w w w. j a v a 2 s . c o m*/ writer.write(new ArrayList<MailboxItem>()); fail("The SQLException got swallowed. It should not be swallowed."); } catch (HiveMailWriterException e) { // This is what we're testing really. // We want to make sure this happens in case of exception. verify(hdfs).delete(any(Path.class), eq(false)); } verify(showStmt).setString(1, "new_table"); verify(loadStmt).setString(eq(1), startsWith("/tmp/timberwolf/")); }
From source file:com.rockstor.compact.TaskMetaWriter.java
License:Apache License
public void create(String path) throws IOException { this.path = path; FileSystem dfs = RockAccessor.getFileSystem(); output = dfs.create(new Path(this.path)); }
From source file:com.shmsoft.dmass.main.MRFreeEedProcess.java
License:Apache License
private void copyToHdfs(String from, String to) throws IOException { Configuration configuration = getConf(); FileSystem fileSystem = FileSystem.get(configuration); // Check if the file already exists Path path = new Path(to); if (fileSystem.exists(path)) { System.out.println("File " + to + " already exists"); return;//from w ww . j a v a2 s . c om } // Create a new file and write data to it. FSDataOutputStream out = fileSystem.create(path); InputStream in = new BufferedInputStream(new FileInputStream(new File(from))); int numBytes = 0; while ((numBytes = in.read(b)) > 0) { out.write(b, 0, numBytes); } // Close all the file descripters in.close(); out.close(); fileSystem.close(); }
From source file:com.singular.utils.FileUtils.java
License:Open Source License
public static Path transferToSystem(Configuration configuration, Path localPath, Path remoteDir, String remoteFileName) {/*from w w w. j a v a2 s. c o m*/ Path finalRemotePath = null; InputStream in = null; OutputStream out = null; System.out.println( "Transferring local " + localPath + " ,remoteStaging " + remoteDir + " ,fileNam=" + remoteFileName); try { FileSystem localFs = FileSystem.getLocal(configuration); FileSystem remoteFs = FileSystem.get(configuration); in = localFs.open(localPath); finalRemotePath = new Path(remoteDir, remoteFileName); out = remoteFs.create(finalRemotePath); copyStream(in, out); } catch (Exception e) { throw new IllegalArgumentException("Exception while transferring file.", e); } finally { if (in != null) { try { in.close(); } catch (IOException e) { e.printStackTrace(); } } if (out != null) { try { out.close(); } catch (IOException e) { e.printStackTrace(); } } } return finalRemotePath; }
From source file:com.singular.utils.FileUtils.java
License:Open Source License
public static Path persistConfigurationInHDFS(Configuration configuration, Properties conf, Path remoteBasePath, String fileName) {/* ww w. j av a 2 s . c om*/ OutputStream out = null; Path remotePath = null; try { remotePath = new Path(remoteBasePath, fileName); FileSystem remoteFs = FileSystem.get(configuration); out = remoteFs.create(remotePath); conf.store(out, null); } catch (Exception e) { e.printStackTrace(); throw new SingularException("Exception while persisting configuration.", e); } finally { try { if (out != null) { out.close(); } } catch (IOException e) { e.printStackTrace(); } } return remotePath; }
From source file:com.singular.utils.FileUtils.java
License:Open Source License
public static OutputStream getPathForStoringConfiguration(Path path, Configuration configuration) throws Exception { FileSystem remoteFs = FileSystem.get(configuration); return remoteFs.create(path); }