List of usage examples for org.apache.hadoop.fs FileSystem append
public FSDataOutputStream append(Path f) throws IOException
From source file:org.apache.flume.sink.customhdfs.HDFSDataStream.java
License:Apache License
protected void doOpen(Configuration conf, Path dstPath, FileSystem hdfs) throws IOException { if (useRawLocalFileSystem) { if (hdfs instanceof LocalFileSystem) { hdfs = ((LocalFileSystem) hdfs).getRaw(); } else {//from w w w .java2s. c om logger.warn("useRawLocalFileSystem is set to true but file system " + "is not of type LocalFileSystem: " + hdfs.getClass().getName()); } } boolean appending = false; if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile(dstPath)) { outStream = hdfs.append(dstPath); appending = true; } else { outStream = hdfs.create(dstPath); } serializer = EventSerializerFactory.getInstance(serializerType, serializerContext, outStream); if (appending && !serializer.supportsReopen()) { outStream.close(); serializer = null; throw new IOException("serializer (" + serializerType + ") does not support append"); } // must call superclass to check for replication issues registerCurrentStream(outStream, hdfs, dstPath); if (appending) { serializer.afterReopen(); } else { serializer.afterCreate(); } }
From source file:org.apache.flume.sink.customhdfs.HDFSSequenceFile.java
License:Apache License
protected void open(Path dstPath, CompressionCodec codeC, CompressionType compType, Configuration conf, FileSystem hdfs) throws IOException { if (useRawLocalFileSystem) { if (hdfs instanceof LocalFileSystem) { hdfs = ((LocalFileSystem) hdfs).getRaw(); } else {//from w ww. j av a 2s . c o m logger.warn("useRawLocalFileSystem is set to true but file system " + "is not of type LocalFileSystem: " + hdfs.getClass().getName()); } } if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile(dstPath)) { outStream = hdfs.append(dstPath); } else { outStream = hdfs.create(dstPath); } writer = SequenceFile.createWriter(conf, outStream, serializer.getKeyClass(), serializer.getValueClass(), compType, codeC); registerCurrentStream(outStream, hdfs, dstPath); }
From source file:org.apache.flume.sink.hdfs.HDFSCompressedDataStream.java
License:Apache License
@Override public void open(String filePath, CompressionCodec codec, CompressionType cType) throws IOException { Configuration conf = new Configuration(); Path dstPath = new Path(filePath); FileSystem hdfs = dstPath.getFileSystem(conf); if (useRawLocalFileSystem) { if (hdfs instanceof LocalFileSystem) { hdfs = ((LocalFileSystem) hdfs).getRaw(); } else {//from w w w .j ava2s . c o m logger.warn("useRawLocalFileSystem is set to true but file system " + "is not of type LocalFileSystem: " + hdfs.getClass().getName()); } } boolean appending = false; if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile(dstPath)) { fsOut = hdfs.append(dstPath); appending = true; } else { fsOut = hdfs.create(dstPath); } cmpOut = codec.createOutputStream(fsOut); serializer = EventSerializerFactory.getInstance(serializerType, serializerContext, cmpOut); if (appending && !serializer.supportsReopen()) { cmpOut.close(); serializer = null; throw new IOException("serializer (" + serializerType + ") does not support append"); } registerCurrentStream(fsOut, hdfs, dstPath); if (appending) { serializer.afterReopen(); } else { serializer.afterCreate(); } isFinished = false; }
From source file:org.apache.gobblin.metrics.GobblinMetrics.java
License:Apache License
private void buildFileMetricReporter(Properties properties) { if (!Boolean.valueOf(properties.getProperty(ConfigurationKeys.METRICS_REPORTING_FILE_ENABLED_KEY, ConfigurationKeys.DEFAULT_METRICS_REPORTING_FILE_ENABLED))) { return;/*from w w w.ja v a 2 s. co m*/ } LOGGER.info("Reporting metrics to log files"); if (!properties.containsKey(ConfigurationKeys.METRICS_LOG_DIR_KEY)) { LOGGER.error("Not reporting metrics to log files because " + ConfigurationKeys.METRICS_LOG_DIR_KEY + " is undefined"); return; } try { String fsUri = properties.getProperty(ConfigurationKeys.FS_URI_KEY, ConfigurationKeys.LOCAL_FS_URI); FileSystem fs = FileSystem.get(URI.create(fsUri), new Configuration()); // Each job gets its own metric log subdirectory Path metricsLogDir = new Path(properties.getProperty(ConfigurationKeys.METRICS_LOG_DIR_KEY), this.getName()); if (!fs.exists(metricsLogDir) && !fs.mkdirs(metricsLogDir)) { LOGGER.error("Failed to create metric log directory for metrics " + this.getName()); return; } // Add a suffix to file name if specified in properties. String metricsFileSuffix = properties.getProperty(ConfigurationKeys.METRICS_FILE_SUFFIX, ConfigurationKeys.DEFAULT_METRICS_FILE_SUFFIX); if (!Strings.isNullOrEmpty(metricsFileSuffix) && !metricsFileSuffix.startsWith(".")) { metricsFileSuffix = "." + metricsFileSuffix; } // Each job run gets its own metric log file Path metricLogFile = new Path(metricsLogDir, this.id + metricsFileSuffix + ".metrics.log"); boolean append = false; // Append to the metric file if it already exists if (fs.exists(metricLogFile)) { LOGGER.info(String.format("Metric log file %s already exists, appending to it", metricLogFile)); append = true; } OutputStream output = append ? fs.append(metricLogFile) : fs.create(metricLogFile, true); // Add metrics reporter OutputStreamReporter.Factory.newBuilder().outputTo(output).build(properties); // Set up events reporter at the same time!! this.codahaleScheduledReporters.add(this.codahaleReportersCloser.register( OutputStreamEventReporter.forContext(RootMetricContext.get()).outputTo(output).build())); LOGGER.info("Will start reporting metrics to directory " + metricsLogDir); } catch (IOException ioe) { LOGGER.error("Failed to build file metric reporter for job " + this.id, ioe); } }
From source file:org.apache.gobblin.metrics.reporter.FileFailureEventReporterTest.java
License:Apache License
@Test public void testReport() throws IOException { MetricContext testContext = MetricContext.builder(getClass().getCanonicalName()).build(); FileSystem fs = mock(FileSystem.class); Path failureLogPath = mock(Path.class); FSDataOutputStream outputStream = mock(FSDataOutputStream.class); FileFailureEventReporter reporter = new FileFailureEventReporter(testContext, fs, failureLogPath); when(fs.exists(any())).thenReturn(true); when(fs.append(any())).thenReturn(outputStream); final String eventName = "testEvent"; final String eventNamespace = "testNamespace"; GobblinTrackingEvent event = new GobblinTrackingEvent(0L, eventNamespace, eventName, Maps.newHashMap()); // Noop on normal event testContext.submitEvent(event);// ww w. java 2 s . co m verify(fs, never()).append(failureLogPath); verify(outputStream, never()).write(anyByte()); // Process failure event FailureEventBuilder failureEvent = new FailureEventBuilder(eventName, eventNamespace); failureEvent.submit(testContext); reporter.report(); // Failure log output is setup verify(fs, times(1)).append(failureLogPath); // Report successfully doAnswer(invocation -> null).when(outputStream).write(any(byte[].class), anyInt(), anyInt()); verify(outputStream, times(1)).write(any(byte[].class), anyInt(), anyInt()); }
From source file:org.apache.ignite.igfs.IgfsLoad.java
License:Apache License
/** * Appends to file.//from w w w .j a v a2 s . co m * * @param fs File system. * @param file File path. * @param appendSize Append size. * @throws IOException If operation failed. */ private static void appendToFile(FileSystem fs, Path file, int appendSize) throws IOException { try (FSDataOutputStream out = fs.append(file)) { out.write(new byte[appendSize]); } }
From source file:org.apache.kylin.monitor.ApiRequestParser.java
License:Apache License
public void parseRequestLog(String filePath, String dPath) throws ParseException, IOException { logger.info("Start parsing kylin api request file " + filePath + " !"); // writer config init FileSystem fs = this.getHdfsFileSystem(); org.apache.hadoop.fs.Path resultStorePath = new org.apache.hadoop.fs.Path(dPath); OutputStreamWriter writer = new OutputStreamWriter(fs.append(resultStorePath)); CSVWriter cwriter = new CSVWriter(writer, '|', CSVWriter.NO_QUOTE_CHARACTER); Pattern p_available = Pattern.compile("/kylin/api/(cubes|user)+.*"); Pattern p_request = Pattern.compile( "^.*\\[.*KylinApiFilter.logRequest.*\\].*REQUEST:.*REQUESTER=(.*);REQ_TIME=(\\w+ (\\d{4}-\\d{2}-\\d{2}).*);URI=(.*);METHOD=(.*);QUERY_STRING=(.*);PAYLOAD=(.*);RESP_STATUS=(.*);$"); Pattern p_uri = Pattern.compile("/kylin/api/(\\w+)(/.*/)*(.*)$"); Matcher m_available = p_available.matcher(""); Matcher m_request = p_request.matcher(""); Matcher m_uri = p_uri.matcher(""); Path path = Paths.get(filePath); try {//from w w w . jav a 2 s . com BufferedReader reader = Files.newBufferedReader(path, ENCODING); String line = null; while ((line = reader.readLine()) != null) { // reset the input m_available.reset(line); m_request.reset(line); // filter unnecessary info if (m_available.find()) { // filter GET info if (m_request.find() && !m_request.group(5).equals("GET")) { List<String> groups = new ArrayList<String>(); for (int i = 1; i <= m_request.groupCount(); i++) { groups.add(m_request.group(i)); } String uri = m_request.group(4); m_uri.reset(uri); if (m_uri.find()) { // add target groups.add(m_uri.group(1)); // add action if (m_uri.group(1).equals("cubes")) { String method = m_request.group(5); if ("DELETE".equals(method)) { groups.add("drop"); } else if ("POST".equals(method)) { groups.add("save"); } else { // add parse action groups.add(m_uri.group(3)); } } } groups.add(DEPLOY_ENV); String[] recordArray = groups.toArray(new String[groups.size()]); // write to hdfs cwriter.writeNext(recordArray); } } } } catch (IOException ex) { logger.info("Failed to write to hdfs:", ex); } finally { writer.close(); cwriter.close(); fs.close(); } logger.info("Finish parsing file " + filePath + " !"); }
From source file:org.apache.kylin.monitor.ApiRequestParser.java
License:Apache License
public void writeResultToHdfs(String dPath, String[] record) throws IOException { OutputStreamWriter writer = null; CSVWriter cwriter = null;/* w w w. j a va 2 s. c om*/ FileSystem fs = null; try { fs = this.getHdfsFileSystem(); org.apache.hadoop.fs.Path resultStorePath = new org.apache.hadoop.fs.Path(dPath); writer = new OutputStreamWriter(fs.append(resultStorePath)); cwriter = new CSVWriter(writer, '|', CSVWriter.NO_QUOTE_CHARACTER); cwriter.writeNext(record); } catch (IOException e) { logger.info("Exception", e); } finally { writer.close(); cwriter.close(); fs.close(); } }
From source file:org.apache.kylin.monitor.FileUtils.java
License:Apache License
public static void appendResultToHdfs(String dPath, String[] record) throws IOException { OutputStreamWriter writer = null; CSVWriter cwriter = null;/*from www .j ava2 s .c o m*/ FileSystem fs = null; try { fs = getHdfsFileSystem(); org.apache.hadoop.fs.Path resultStorePath = new org.apache.hadoop.fs.Path(dPath); writer = new OutputStreamWriter(fs.append(resultStorePath)); cwriter = new CSVWriter(writer, '|', CSVWriter.NO_QUOTE_CHARACTER); cwriter.writeNext(record); } catch (Exception e) { logger.info("Exception", e); } finally { if (writer != null) { writer.close(); } if (cwriter != null) { cwriter.close(); } if (fs != null) { fs.close(); } } }
From source file:org.apache.kylin.monitor.QueryParser.java
License:Apache License
public void parseQueryLog(String filePath, String dPath) throws ParseException, IOException { logger.info("Start parsing file " + filePath + " !"); // writer config init FileSystem fs = this.getHdfsFileSystem(); org.apache.hadoop.fs.Path resultStorePath = new org.apache.hadoop.fs.Path(dPath); OutputStreamWriter writer = new OutputStreamWriter(fs.append(resultStorePath)); CSVWriter cwriter = new CSVWriter(writer, '|', CSVWriter.NO_QUOTE_CHARACTER); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS"); Pattern p_query_start = Pattern.compile("^\\[.*\\]:\\[(.*),.*\\]\\[.*\\]\\[.*QueryService.logQuery.*\\].*"); Pattern p_query_end = Pattern.compile("^Message:(.*)$"); Pattern p_query_body = Pattern.compile( "^\\[.*\\]:\\[((\\d{4}-\\d{2}-\\d{2}).*)\\]\\[.*\\]\\[.*\\].*\n^=+\\[QUERY\\]=+\n^SQL:(.*)\n^User:(.*)\n^Success:(.*)\n^Duration:(.*)\n^Project:(.*)\n^(Realization Names|Cube Names): \\[(.*)\\]\n^Cuboid Ids: \\[(.*)\\]\n^Total scan count:(.*)\n^Result row count:(.*)\n^Accept Partial:(.*)\n(^Is Partial Result:(.*)\n)?^Hit Cache:(.*)\n^Message:(.*)", Pattern.MULTILINE);//from ww w . ja v a2s. c om Matcher m_query_start = p_query_start.matcher(""); Matcher m_query_end = p_query_end.matcher(""); Matcher m_query_body = p_query_body.matcher(""); boolean query_start = false; StringBuffer query_body = new StringBuffer(""); Path path = Paths.get(filePath); try { BufferedReader reader = Files.newBufferedReader(path, ENCODING); String line = null; while ((line = reader.readLine()) != null) { m_query_start.reset(line); //reset the input m_query_end.reset(line); // set start flag ,clear StringBuffer if (m_query_start.find()) { query_start = true; query_body = new StringBuffer(""); } if (query_start) { query_body.append(line + "\n"); } if (m_query_end.find()) { query_start = false; m_query_body.reset(query_body); logger.info("parsing query..."); logger.info(query_body); // skip group(8) and group(14) if (m_query_body.find()) { ArrayList<String> groups = new ArrayList<String>(); int grp_count = m_query_body.groupCount(); for (int i = 1; i <= grp_count; i++) { if (i != 8 && i != 14) { String grp_item = m_query_body.group(i); grp_item = grp_item == null ? "" : grp_item.trim(); groups.add(grp_item); } } long start_time = format.parse(groups.get(0)).getTime() - (int) (Double.parseDouble(groups.get(5)) * 1000); groups.set(0, format.format(new Date(start_time))); groups.add(DEPLOY_ENV); String[] recordArray = groups.toArray(new String[groups.size()]); // write to hdfs cwriter.writeNext(recordArray); } } } } catch (IOException ex) { logger.info("Failed to write to hdfs:", ex); } finally { if (writer != null) { writer.close(); } if (cwriter != null) { cwriter.close(); } if (fs != null) { fs.close(); } } logger.info("Finish parsing file " + filePath + " !"); }