List of usage examples for org.apache.hadoop.fs FileSystem create
public FSDataOutputStream create(Path f) throws IOException
From source file:com.asakusafw.runtime.directio.hadoop.DirectIoTransactionEditorTest.java
License:Apache License
private void indoubt(String executionId) throws IOException, InterruptedException { Path txPath = HadoopDataSourceUtil.getTransactionInfoPath(conf, executionId); Path cmPath = HadoopDataSourceUtil.getCommitMarkPath(conf, executionId); FileSystem fs = txPath.getFileSystem(conf); fs.create(txPath).close(); fs.create(cmPath).close();// w w w .java 2 s. com int index = 0; for (String path : repo.getContainerPaths()) { String id = repo.getRelatedId(path); DirectDataSource ds = repo.getRelatedDataSource(path); OutputTransactionContext txContext = HadoopDataSourceUtil.createContext(executionId, id); OutputAttemptContext aContext = new OutputAttemptContext(txContext.getTransactionId(), String.valueOf(index), txContext.getOutputId(), new Counter()); ds.setupTransactionOutput(txContext); ds.setupAttemptOutput(aContext); try (ModelOutput<StringBuilder> output = ds.openOutput(aContext, SimpleDataDefinition.newInstance(StringBuilder.class, new MockFormat()), "", executionId, new Counter())) { output.write(new StringBuilder("Hello, world!")); } ds.commitAttemptOutput(aContext); ds.cleanupAttemptOutput(aContext); index++; } }
From source file:com.asakusafw.runtime.directio.hadoop.HadoopFileFormatAdapter.java
License:Apache License
@Override public ModelOutput<T> createOutput(Class<? extends T> dataType, FileSystem fileSystem, final Path path, Counter counter) throws IOException, InterruptedException { FSDataOutputStream stream = fileSystem.create(path); boolean succeed = false; try {/*from w ww. ja v a 2 s.c o m*/ CountOutputStream cstream; if (LOG.isDebugEnabled()) { cstream = new CountOutputStream(stream, counter) { @Override public void close() throws IOException { LOG.debug(MessageFormat.format("Start closing output (file={0})", //$NON-NLS-1$ path)); super.close(); LOG.debug(MessageFormat.format("Finish closing output (file={0})", //$NON-NLS-1$ path)); } }; } else { cstream = new CountOutputStream(stream, counter); } ModelOutput<T> output = streamFormat.createOutput(dataType, path.toString(), cstream); succeed = true; return output; } finally { if (succeed == false) { try { stream.close(); } catch (IOException e) { LOG.warn(MessageFormat.format("Failed to close output (path={0})", path), e); } } } }
From source file:com.avira.couchdoop.imp.PageFileWriter.java
License:Apache License
public PageFileWriter(Configuration conf, String dirName, String baseName, int page) throws IOException { String destinationPath;/*w w w . j av a 2s . c o m*/ if (dirName.isEmpty()) { // If no dirName is provided write the file in current directory. destinationPath = baseName + "-" + String.format("%05d", page); } else { destinationPath = dirName + "/" + baseName + "-" + String.format("%05d", page); } // Get the file system. FileSystem fileSystem = FileSystem.get(URI.create(destinationPath), conf); // Get an OutputStream for the output file. Path path = new Path(destinationPath); LOGGER.info("Creating file '" + path + "'..."); outputStream = fileSystem.create(path); }
From source file:com.benchmark.mapred.dancing.DistributedPentomino.java
License:Apache License
/** * Create the input file with all of the possible combinations of the * given depth.// w w w . j a v a2 s. c o m * @param fs the filesystem to write into * @param dir the directory to write the input file into * @param pent the puzzle * @param depth the depth to explore when generating prefixes */ private static void createInputDirectory(FileSystem fs, Path dir, Pentomino pent, int depth) throws IOException { fs.mkdirs(dir); List<int[]> splits = pent.getSplits(depth); PrintStream file = new PrintStream(new BufferedOutputStream(fs.create(new Path(dir, "part1")), 64 * 1024)); for (int[] prefix : splits) { for (int i = 0; i < prefix.length; ++i) { if (i != 0) { file.print(','); } file.print(prefix[i]); } file.print('\n'); } file.close(); }
From source file:com.bigdog.hadoop.hdfs.HDFS_Test.java
public void writeText() { String str = "Hello world,this is my first hdfs app\n"; java.io.OutputStream out;/* w w w . j av a 2 s . co m*/ try { Configuration config = new Configuration(); FileSystem fs = FileSystem.get(config); out = fs.create(new Path(Constants.output + "helloworld.txt")); out.write(str.getBytes()); out.flush(); out.close(); } catch (IOException ex) { Logger.getLogger(HDFS_Test.class.getName()).log(Level.SEVERE, null, ex); } }
From source file:com.bigdog.hadoop.hdfs.HDFS_Test.java
public void createNewHDFSFile(String toCreateFilePath, String content) throws IOException { Configuration config = new Configuration(); FileSystem hdfs = FileSystem.get(config); FSDataOutputStream os = hdfs.create(new Path(toCreateFilePath)); os.write(content.getBytes("UTF-8")); os.close();/*w w w .jav a 2s .c o m*/ hdfs.close(); }
From source file:com.blackberry.logdriver.util.IndexLogs.java
License:Apache License
public static void main(String args[]) throws IOException, ParseException { // Create blank map for components Map<String, Map<String, Map<String, Map<String, Component>>>> data = new HashMap<String, Map<String, Map<String, Map<String, Component>>>>(); List<String> unmergedCSVStrings = new ArrayList<String>(); unmergedCSVStrings.add("DC,Service,Type,Component,Date,Hour\n"); // Set the output format Boolean humanReadable = false; Boolean writeIndex = true;/* w ww .j a v a 2s.c o m*/ Boolean removeOldIndexes = false; Boolean forceRemove = false; Boolean confirmRemoval = false; for (int i = 0; i < args.length; i++) { if (args[i].matches("-t")) { humanReadable = true; } else if (args[i].matches("-n")) { writeIndex = false; } else if (args[i].matches("-r")) { removeOldIndexes = true; } else if (args[i].matches("-f")) { forceRemove = true; } else { System.out.println( "Usage: indexlogs [-t -n -r -f]\n -t Print results to STDOUT in human-readable tree\n" + " -n Don't write index files into HDFS\n" + " -r Remove old index files from HDFS\n" + " -f Force remove old index files (requires -r)"); System.exit(0); } } // Set up HDFS filesystem Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); // Search the /service folder for matching paths if (!humanReadable && !writeIndex) { System.out.println("Warning: -n set without -t, not doing anything.\n"); System.exit(0); } if (forceRemove && !removeOldIndexes) { System.out.println( "Warning: Asked to force-remove (-f) indexes without removing old index files (-r). Aborting."); System.exit(0); } //Confirm removal of old IndexLogs files if (removeOldIndexes) { if (!forceRemove) { Scanner reader = new Scanner(System.in); String input; boolean validAction = false; do { System.out.println("Delete *all* IndexLogs files from HDFS? [y/n]"); input = reader.nextLine(); if (input.startsWith("y")) { validAction = true; confirmRemoval = true; } } while (!validAction && !(input.startsWith("n"))); reader.close(); } } System.out.println("Indexing logs..."); findComponents(data, unmergedCSVStrings, fs, new Path("/service")); // Remove old IndexLogs files if (confirmRemoval || forceRemove) { System.out.println("Removing all old IndexLogs Files from HDFS..."); //WARNING: DO NOT CHANGE THIS UNLESS ABSOLUTELY NECESSARY. fs.delete(new Path("/service/.index/"), true); } else { System.out.println("Not removing old IndexLogs files."); } // Output the generated index if (humanReadable) { humanPrint(data); System.out.println(""); } if (writeIndex) { long currentTime = System.currentTimeMillis() / 1000; FSDataOutputStream outputCSV = fs.create(new Path("/service/.index/logindex." + currentTime + ".csv")); writeCSV(data, outputCSV); outputCSV.close(); FSDataOutputStream outputJSON = fs .create(new Path("/service/.index/logindex." + currentTime + ".json")); writeJSON(data, outputJSON); outputJSON.close(); System.out.println("Index files written to /service/.index/logindex." + currentTime + ".csv and /service/.index/logindex." + currentTime + ".json"); FSDataOutputStream unmergedCSV = fs .create(new Path("/service/.index/unmerged." + currentTime + ".csv")); Iterator<String> i = unmergedCSVStrings.iterator(); while (i.hasNext()) { unmergedCSV.writeBytes(i.next()); } unmergedCSV.close(); System.out.println("Unmerged report written to /service/.index/unmerged." + currentTime + ".csv"); } }
From source file:com.blackberry.logtools.logmultisearch.java
License:Apache License
@SuppressWarnings("static-access") public int run(String[] argv) throws Exception { //Configuring configuration and filesystem to work on HDFS final Configuration conf = getConf(); //Configuration processed by ToolRunner FileSystem fs = FileSystem.get(conf); //Initiate tools used for running search LogTools tools = new LogTools(); //Other options String date_format = "RFC5424"; String field_separator = " "; ArrayList<String> D_options = new ArrayList<String>(); boolean quiet = true; boolean silent = false; boolean log = false; boolean forcelocal = false; boolean forceremote = false; //The arguments are // - strings// w w w . j a v a2 s . c o m // - dc number // - service // - component // - startTime (Something 'date' can parse, or just a time in ms from epoch) // - endTime (Same as start) // - outputDir //Indexing for arguments to be passed for Mapreduce int stringsNum = 0; int dcNum = 1; int svcNum = 2; int compNum = 3; int startNum = 4; int endNum = 5; int outNum = 6; //Parsing through user arguments String[] args = new String[7]; int count = 0; //Count created to track the parse of all arguments int argcount = 0; //Count created to track number of arguments to be passed on while (count < argv.length) { String arg = argv[count]; count++; if (arg.equals("--")) { break; } else if (arg.startsWith("-")) { if (arg.equals("--v")) { quiet = tools.parseV(silent); } else if (arg.equals("--i")) { conf.set("logdriver.search.case.insensitive", "true"); } else if (arg.equals("--a")) { LogTools.logConsole(quiet, silent, warn, "AND searching selected"); conf.set("logdriver.search.and", "true"); } else if (arg.startsWith("--dateFormat=")) { arg = arg.replace("--dateFormat=", ""); date_format = arg; } else if (arg.startsWith("--fieldSeparator=")) { arg = arg.replace("--fieldSeparator=", ""); field_separator = arg; } else if (arg.startsWith("-strings=")) { arg = arg.replace("-strings=", ""); args[stringsNum] = arg; argcount++; } else if (arg.startsWith("-dc=")) { arg = arg.replace("-dc=", ""); args[dcNum] = arg; argcount++; } else if (arg.startsWith("-svc=")) { arg = arg.replace("-svc=", ""); args[svcNum] = arg; argcount++; } else if (arg.startsWith("-comp=")) { arg = arg.replace("-comp=", ""); args[compNum] = arg; argcount++; } else if (arg.startsWith("-start=")) { arg = arg.replace("-start=", ""); args[startNum] = arg; argcount++; } else if (arg.startsWith("-end=")) { arg = arg.replace("-end=", ""); args[endNum] = arg; argcount++; } //User inputs output directory that is to be created //Check to see if parent directory exists && output directory does not exist else if (arg.startsWith("--out=")) { args[outNum] = tools.parseOut(arg, fs); argcount++; } else if (arg.startsWith("-D")) { D_options.add(arg); } else if (arg.equals("--silent")) { silent = tools.parseSilent(quiet); } else if (arg.equals("--log")) { log = true; } else if (arg.equals("--l")) { forcelocal = tools.parsePigMode(forceremote); } else if (arg.equals("--r")) { forceremote = tools.parsePigMode(forcelocal); } else { LogTools.logConsole(quiet, silent, error, "Unrecognized option: " + arg); System.exit(1); } } else { LogTools.logConsole(quiet, silent, error, "Unrecognized option: " + arg); System.exit(1); } } //Default output should be stdout represented by "-" if (args[outNum] == null) { args[outNum] = "-"; argcount++; LogTools.logConsole(quiet, silent, info, "Output set to default stdout."); } if (argcount < 7) { System.err.println(";****************************************" + "\n\t\t\t NOT ENOUGH ARGUMENTS\n" + "\n\tUSAGE: logmultisearch [REQUIRED ARGUMENTS] [OPTIONS] (Order does not matter)" + "\n\tREQUIRED ARGUMENTS:" + "\n\t\t-strings=[STRINGS_DIR|STRINGS_FILE|STRING] String/file/directory of strings to search." + "\n\t\t-dc=[DATACENTER] Data Center." + "\n\t\t-svc=[SERVICE] Service." + "\n\t\t-comp=[COMPONENT] Component." + "\n\t\t-start=[START] Start time." + "\n\t\t-end=[END] End time." + "\n\tOptions:" + "\n\t\t--out=[DIRECTORY] Desired output directory. If not defined, output to stdout." + "\n\t\t--v Verbose output." + "\n\t\t--r Force remote sort." + "\n\t\t--l Force local sort." + "\n\t\t--dateFormat=[FORMAT] Valid formats are RFC822, RFC3164 (zero padded day)," + "\n\t RFC5424 (default), or any valid format string for FastDateFormat." + "\n\t\t--fieldSeparator=X The separator to use to separate fields in intermediate" + "\n\t files. Defaults to 'INFORMATION SEPARATOR ONE' (U+001F)." + "\n\t\t--silent Output only the data." + "\n\t\t--i Make search case insensitive." + "\n\t\t--a Enable AND searching." + "\n\t\t--log Save all the logs.\n" + ";****************************************"); System.exit(1); } //Parse time inputs for start and end of search args[startNum] = tools.parseDate(args[startNum]); args[endNum] = tools.parseDate(args[endNum]); tools.checkTime(args[startNum], args[endNum]); //Retrieve 'strings' argument to be able to pass search strings to HDFS //Retrieve 'out' argument to determine where output of results should be sent String strings = args[stringsNum]; String out = args[outNum]; //Generate files to temporarily store output of mapreduce jobs and pig logs locally File local_output = File.createTempFile("tmp.", RandomStringUtils.randomAlphanumeric(10)); if (log != true) { local_output.deleteOnExit(); } File pig_tmp = File.createTempFile("tmp.", RandomStringUtils.randomAlphanumeric(10)); if (log != true) { pig_tmp.deleteOnExit(); } //Name the temp directory for storing results in HDFS String tmp = "tmp/logmultisearch-" + RandomStringUtils.randomAlphanumeric(10); //Set args[stringsNum] to be location of search strings to be used for the Multisearch args[stringsNum] = (StringEscapeUtils.escapeJava(tmp) + "/strings"); //Set args[outNum] to be temp output directory to be passed onto MultiSearchByTime instead of UserInput argument args[outNum] = (StringEscapeUtils.escapeJava(tmp) + "/rawlines"); //Managing console output - deal with --v/--silent Logger LOG = LoggerFactory.getLogger(logmultisearch.class); tools.setConsoleOutput(local_output, quiet, silent); //Create temp directory in HDFS to store logsearch logs before sorting tools.tmpDirHDFS(quiet, silent, fs, conf, tmp, log); //If the strings argument is the path of a file, copy the file to HDFS. //If the strings argument is the path of a directory, copy all files in the directory to HDFS. //If the strings argument is not a path to a file/directory, write to a newly created file in HDFS. try { File f = new File(strings); if (f.isFile()) { LogTools.logConsole(quiet, silent, warn, "Strings input is a File..."); //dos2unix file conversion File dos2unix = File.createTempFile("tmp.", RandomStringUtils.randomAlphanumeric(10)); dos2unix.deleteOnExit(); tools.dosTounix(f, dos2unix); //Copy over temp directory into a new directory in HDFS to be used for logmultisearch fs.copyFromLocalFile(new Path(dos2unix.getAbsolutePath()), new Path(tmp + "/strings")); } else if (f.isDirectory()) { LogTools.logConsole(quiet, silent, warn, "Strings input is a Directory..."); //Get list of all files in directory to convert from dos2unix String[] fileList = f.list(); //Create temp directory to store all converted files File tempDir = Files.createTempDir(); tempDir.deleteOnExit(); //Convert all files from dos2unix and write to temp directory for (int i = 0; i < fileList.length; i++) { File dos2unix = File.createTempFile("unix", fileList[i], tempDir); dos2unix.deleteOnExit(); tools.dosTounix(new File(f.getAbsolutePath() + "/" + fileList[i]), dos2unix); } //Copy over temp directory into a new directory in HDFS to be used for logmultisearch fs.copyFromLocalFile(new Path(tempDir.getAbsolutePath()), new Path(tmp + "/strings")); } else { LogTools.logConsole(quiet, silent, warn, "Strings input is a search string..."); //Make directory and file for strings fs.mkdirs(new Path(tmp + "/strings")); fs.createNewFile(new Path(tmp + "/strings/strings")); //Write search strings to file FSDataOutputStream hdfsOut = fs.create(new Path(tmp + "/strings/strings")); hdfsOut.writeUTF(strings); hdfsOut.close(); } } catch (Exception e) { e.printStackTrace(); System.exit(1); } LogTools.logConsole(quiet, silent, warn, "Searching..."); LogTools.logConsole(quiet, silent, warn, "Passing Arguments: SearchStrings=" + strings + " DC=" + args[dcNum] + " Service=" + args[svcNum] + " Component=" + args[compNum] + " StartTime=" + args[startNum] + " EndTime=" + args[endNum] + " Output=" + out); //Set standard configuration for running Mapreduce and PIG String queue_name = "logsearch"; //Start Mapreduce job tools.runMRJob(quiet, silent, conf, D_options, out, LOG, field_separator, queue_name, args, "MultiSearchByTime", new MultiSearchByTime()); //Before sorting, determine the number of records and size of the results found long foundresults = tools.getResults(local_output); long size = tools.getSize(foundresults, tmp, fs); //Run PIG job if results found tools.runPig(silent, quiet, foundresults, size, tmp, out, D_options, queue_name, date_format, field_separator, pig_tmp, fs, conf, forcelocal, forceremote); //Display location of tmp files if log enabled tools.logs(log, local_output, pig_tmp, tmp); return 0; }
From source file:com.chinamobile.bcbsp.bspstaff.BSPStaff.java
License:Apache License
private void writeMigratePartition(HashMap<String, Integer> migrateMap, StaffSSControllerInterface sssc) throws IOException { String migratePartitionString = this.migrateDirBase + "/" + String.valueOf(this.currentSuperStepCounter / Constants.K) + "/" + String.valueOf(this.getPartition()); LOG.info("ljn test : writeMigratePartition " + migratePartitionString); this.migratePartitionDir = this.migrateDirBase + "/" + String.valueOf(this.currentSuperStepCounter / Constants.K); Path migratePartitionPath = new Path(migratePartitionString); FileSystem fsFileSystem = FileSystem.get(this.getConf().getConf()); FSDataOutputStream fsOutput = fsFileSystem.create(migratePartitionPath); for (String vid : migrateMap.keySet()) { // LOG.info("ljn test : write " + vid + " :" + migrateMap.get(vid)); String str = vid + ":" + migrateMap.get(vid) + "\n"; fsOutput.write(str.getBytes("UTF-8")); // Text a = new Text(vid + ":" + migrateMap.get(vid)); // a.write(fsOutput); }/*from w ww .j av a2 s . c o m*/ fsFileSystem.close(); }
From source file:com.citic.zxyjs.zwlscx.mapreduce.lib.input.HFileOutputFormatBase.java
License:Apache License
public RecordWriter<ImmutableBytesWritable, KeyValue> getRecordWriter(final TaskAttemptContext context) throws IOException, InterruptedException { // Get the path of the temporary output file final Path outputPath = FileOutputFormat.getOutputPath(context); final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath(); final Path ignoreOutputPath = new Path(outputPath + "_ignore"); final Configuration conf = context.getConfiguration(); final FileSystem fs = outputdir.getFileSystem(conf); // These configs. are from hbase-*.xml final long maxsize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, HConstants.DEFAULT_MAX_FILE_SIZE); // Invented config. Add to hbase-*.xml if other than default // compression. final String defaultCompression = conf.get("hfile.compression", Compression.Algorithm.NONE.getName()); final boolean compactionExclude = conf.getBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude", false);//w ww .j a va 2 s . co m if (fs.exists(ignoreOutputPath)) { LOG.info("Deleted " + ignoreOutputPath.toString() + " success."); fs.delete(ignoreOutputPath, true); } // create a map from column family to the compression algorithm final Map<byte[], String> compressionMap = createFamilyCompressionMap(conf); final Map<byte[], String> bloomTypeMap = createFamilyBloomMap(conf); final Map<byte[], String> blockSizeMap = createFamilyBlockSizeMap(conf); String dataBlockEncodingStr = conf.get(DATABLOCK_ENCODING_CONF_KEY); final HFileDataBlockEncoder encoder; if (dataBlockEncodingStr == null) { encoder = NoOpDataBlockEncoder.INSTANCE; } else { try { encoder = new HFileDataBlockEncoderImpl(DataBlockEncoding.valueOf(dataBlockEncodingStr)); } catch (IllegalArgumentException ex) { throw new RuntimeException("Invalid data block encoding type configured for the param " + DATABLOCK_ENCODING_CONF_KEY + " : " + dataBlockEncodingStr); } } return new RecordWriter<ImmutableBytesWritable, KeyValue>() { // Map of families to writers and how much has been output on the // writer. private final Map<byte[], WriterLength> writers = new TreeMap<byte[], WriterLength>( Bytes.BYTES_COMPARATOR); private final FSDataOutputStream dos = fs.create(ignoreOutputPath); private byte[] previousRow = HConstants.EMPTY_BYTE_ARRAY; private final byte[] now = Bytes.toBytes(System.currentTimeMillis()); private boolean rollRequested = false; public void write(ImmutableBytesWritable row, KeyValue kv) throws IOException { // null input == user explicitly wants to flush if (row == null && kv == null) { rollWriters(); return; } byte[] rowKey = kv.getRow(); long length = kv.getLength(); byte[] family = kv.getFamily(); if (ignore(kv)) { byte[] readBuf = rowKey; dos.write(readBuf, 0, readBuf.length); dos.write(Bytes.toBytes("\n")); return; } WriterLength wl = this.writers.get(family); // If this is a new column family, verify that the directory // exists if (wl == null) { Path path = null; path = new Path(outputdir, Bytes.toString(family)); fs.mkdirs(path); } // If any of the HFiles for the column families has reached // maxsize, we need to roll all the writers if (wl != null && wl.written + length >= maxsize) { this.rollRequested = true; } // This can only happen once a row is finished though if (rollRequested && Bytes.compareTo(this.previousRow, rowKey) != 0) { rollWriters(); } // create a new HLog writer, if necessary if (wl == null || wl.writer == null) { wl = getNewWriter(family, conf); } // we now have the proper HLog writer. full steam ahead kv.updateLatestStamp(this.now); wl.writer.append(kv); wl.written += length; // Copy the row so we know when a row transition. this.previousRow = rowKey; } private void rollWriters() throws IOException { for (WriterLength wl : this.writers.values()) { if (wl.writer != null) { LOG.info("Writer=" + wl.writer.getPath() + ((wl.written == 0) ? "" : ", wrote=" + wl.written)); close(wl.writer); } wl.writer = null; wl.written = 0; } this.rollRequested = false; } /* * Create a new StoreFile.Writer. * @param family * @return A WriterLength, containing a new StoreFile.Writer. * @throws IOException */ private WriterLength getNewWriter(byte[] family, Configuration conf) throws IOException { WriterLength wl = new WriterLength(); Path familydir = new Path(outputdir, Bytes.toString(family)); String compression = compressionMap.get(family); compression = compression == null ? defaultCompression : compression; String bloomTypeStr = bloomTypeMap.get(family); BloomType bloomType = BloomType.NONE; if (bloomTypeStr != null) { bloomType = BloomType.valueOf(bloomTypeStr); } String blockSizeString = blockSizeMap.get(family); int blockSize = blockSizeString == null ? HConstants.DEFAULT_BLOCKSIZE : Integer.parseInt(blockSizeString); Configuration tempConf = new Configuration(conf); tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f); wl.writer = new StoreFile.WriterBuilder(conf, new CacheConfig(tempConf), fs, blockSize) .withOutputDir(familydir) .withCompression(AbstractHFileWriter.compressionByName(compression)) .withBloomType(bloomType).withComparator(KeyValue.COMPARATOR).withDataBlockEncoder(encoder) .withChecksumType(HStore.getChecksumType(conf)) .withBytesPerChecksum(HStore.getBytesPerChecksum(conf)).build(); this.writers.put(family, wl); return wl; } private void close(final StoreFile.Writer w) throws IOException { if (w != null) { w.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis())); w.appendFileInfo(StoreFile.BULKLOAD_TASK_KEY, Bytes.toBytes(context.getTaskAttemptID().toString())); w.appendFileInfo(StoreFile.MAJOR_COMPACTION_KEY, Bytes.toBytes(true)); w.appendFileInfo(StoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY, Bytes.toBytes(compactionExclude)); w.appendTrackedTimestampsToMetadata(); w.close(); } } public void close(TaskAttemptContext c) throws IOException, InterruptedException { dos.flush(); dos.close(); for (WriterLength wl : this.writers.values()) { close(wl.writer); } } }; }