Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package gobblin.publisher; import java.io.IOException; import java.net.URI; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.io.Closer; import gobblin.configuration.ConfigurationKeys; import gobblin.configuration.State; import gobblin.configuration.WorkUnitState; import gobblin.metadata.MetadataMerger; import gobblin.util.ForkOperatorUtils; import gobblin.util.HadoopUtils; import gobblin.util.ParallelRunner; import gobblin.util.WriterUtils; import gobblin.util.reflection.GobblinConstructorUtils; /** * A basic implementation of {@link SingleTaskDataPublisher} that publishes the data from the writer output directory * to the final output directory. * * <p> * The final output directory is specified by {@link ConfigurationKeys#DATA_PUBLISHER_FINAL_DIR}. The output of each * writer is written to this directory. Each individual writer can also specify a path in the config key * {@link ConfigurationKeys#WRITER_FILE_PATH}. Then the final output data for a writer will be * {@link ConfigurationKeys#DATA_PUBLISHER_FINAL_DIR}/{@link ConfigurationKeys#WRITER_FILE_PATH}. If the * {@link ConfigurationKeys#WRITER_FILE_PATH} is not specified, a default one is assigned. The default path is * constructed in the {@link gobblin.source.workunit.Extract#getOutputFilePath()} method. * </p> * * <p> * This publisher records all dirs it publishes to in property {@link ConfigurationKeys#PUBLISHER_DIRS}. Each time it * publishes a {@link Path}, if the path is a directory, it records this path. If the path is a file, it records the * parent directory of the path. To change this behavior one may override * {@link #recordPublisherOutputDirs(Path, Path, int)}. * </p> */ public class BaseDataPublisher extends SingleTaskDataPublisher { private static final Logger LOG = LoggerFactory.getLogger(BaseDataPublisher.class); protected final int numBranches; protected final List<FileSystem> writerFileSystemByBranches; protected final List<FileSystem> publisherFileSystemByBranches; protected final List<FileSystem> metaDataWriterFileSystemByBranches; protected final List<Optional<String>> publisherFinalDirOwnerGroupsByBranches; protected final List<FsPermission> permissions; protected final Closer closer; protected final Closer parallelRunnerCloser; protected final int parallelRunnerThreads; protected final Map<String, ParallelRunner> parallelRunners = Maps.newHashMap(); protected final Set<Path> publisherOutputDirs = Sets.newHashSet(); protected final List<MetadataMerger<String>> metadataMergers; public BaseDataPublisher(State state) throws IOException { super(state); this.closer = Closer.create(); Configuration conf = new Configuration(); // Add all job configuration properties so they are picked up by Hadoop for (String key : this.getState().getPropertyNames()) { conf.set(key, this.getState().getProp(key)); } this.numBranches = this.getState().getPropAsInt(ConfigurationKeys.FORK_BRANCHES_KEY, 1); this.writerFileSystemByBranches = Lists.newArrayListWithCapacity(this.numBranches); this.publisherFileSystemByBranches = Lists.newArrayListWithCapacity(this.numBranches); this.metaDataWriterFileSystemByBranches = Lists.newArrayListWithCapacity(this.numBranches); this.publisherFinalDirOwnerGroupsByBranches = Lists.newArrayListWithCapacity(this.numBranches); this.permissions = Lists.newArrayListWithCapacity(this.numBranches); this.metadataMergers = mergersForEachBranch(); // Get a FileSystem instance for each branch for (int i = 0; i < this.numBranches; i++) { URI writerUri = URI.create(this.getState().getProp(ForkOperatorUtils .getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, this.numBranches, i), ConfigurationKeys.LOCAL_FS_URI)); this.writerFileSystemByBranches.add(FileSystem.get(writerUri, conf)); URI publisherUri = URI .create(this.getState().getProp( ForkOperatorUtils.getPropertyNameForBranch( ConfigurationKeys.DATA_PUBLISHER_FILE_SYSTEM_URI, this.numBranches, i), writerUri.toString())); this.publisherFileSystemByBranches.add(FileSystem.get(publisherUri, conf)); this.metaDataWriterFileSystemByBranches.add(FileSystem.get(publisherUri, conf)); // The group(s) will be applied to the final publisher output directory(ies) this.publisherFinalDirOwnerGroupsByBranches .add(Optional.fromNullable(this.getState().getProp(ForkOperatorUtils.getPropertyNameForBranch( ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR_GROUP, this.numBranches, i)))); // The permission(s) will be applied to all directories created by the publisher, // which do NOT include directories created by the writer and moved by the publisher. // The permissions of those directories are controlled by writer.file.permissions and writer.dir.permissions. this.permissions.add(new FsPermission(state.getPropAsShortWithRadix( ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.DATA_PUBLISHER_PERMISSIONS, this.numBranches, i), FsPermission.getDefault().toShort(), ConfigurationKeys.PERMISSION_PARSING_RADIX))); } this.parallelRunnerThreads = state.getPropAsInt(ParallelRunner.PARALLEL_RUNNER_THREADS_KEY, ParallelRunner.DEFAULT_PARALLEL_RUNNER_THREADS); this.parallelRunnerCloser = Closer.create(); } // Build the list of metadataMergers for each branch. Return null if none are configured at all private List<MetadataMerger<String>> mergersForEachBranch() { List<MetadataMerger<String>> metadataMergers = new ArrayList<>(); boolean mergersConfigured = false; for (int i = 0; i < numBranches; i++) { if (shouldPublishMetadataForBranch(i)) { metadataMergers.add(buildPublisherForBranch(i)); mergersConfigured = true; } else { metadataMergers.add(null); } } return mergersConfigured ? metadataMergers : null; } private MetadataMerger<String> buildPublisherForBranch(int branchId) { String keyName = ForkOperatorUtils.getPropertyNameForBranch( ConfigurationKeys.DATA_PUBLISH_WRITER_METADATA_MERGER_NAME_KEY, this.numBranches, branchId); String className = this.getState().getProp(keyName, ConfigurationKeys.DATA_PUBLISH_WRITER_METADATA_MERGER_NAME_DEFAULT); try { Class<?> mdClass = Class.forName(className); // If the merger understands properties, use that constructor; otherwise use the default // parameter-less ctor @SuppressWarnings("unchecked") Object merger = GobblinConstructorUtils.invokeFirstConstructor(mdClass, Collections.<Object>singletonList(this.getState().getProperties()), Collections.<Object>emptyList()); try { @SuppressWarnings("unchecked") MetadataMerger<String> casted = (MetadataMerger<String>) merger; return casted; } catch (ClassCastException e) { throw new IllegalArgumentException(className + " does not implement the MetadataMerger interface", e); } } catch (ClassNotFoundException e) { throw new IllegalArgumentException("Specified metadata merger class " + className + " not found!", e); } catch (ReflectiveOperationException e) { throw new IllegalArgumentException("Error building merger class " + className, e); } } @Override public void initialize() throws IOException { // Nothing needs to be done since the constructor already initializes the publisher. } @Override public void close() throws IOException { try { for (Path path : this.publisherOutputDirs) { this.state.appendToSetProp(ConfigurationKeys.PUBLISHER_DIRS, path.toString()); } } finally { this.closer.close(); } } @Override public void publishData(WorkUnitState state) throws IOException { for (int branchId = 0; branchId < this.numBranches; branchId++) { publishSingleTaskData(state, branchId); } this.parallelRunnerCloser.close(); } /** * This method publishes output data for a single task based on the given {@link WorkUnitState}. * Output data from other tasks won't be published even if they are in the same folder. */ private void publishSingleTaskData(WorkUnitState state, int branchId) throws IOException { publishData(state, branchId, true, new HashSet<Path>()); } @Override public void publishData(Collection<? extends WorkUnitState> states) throws IOException { // We need a Set to collect unique writer output paths as multiple tasks may belong to the same extract. Tasks that // belong to the same Extract will by default have the same output directory Set<Path> writerOutputPathsMoved = Sets.newHashSet(); for (WorkUnitState workUnitState : states) { for (int branchId = 0; branchId < this.numBranches; branchId++) { publishMultiTaskData(workUnitState, branchId, writerOutputPathsMoved); } } this.parallelRunnerCloser.close(); for (WorkUnitState workUnitState : states) { // Upon successfully committing the data to the final output directory, set states // of successful tasks to COMMITTED. leaving states of unsuccessful ones unchanged. // This makes sense to the COMMIT_ON_PARTIAL_SUCCESS policy. workUnitState.setWorkingState(WorkUnitState.WorkingState.COMMITTED); } } /** * This method publishes task output data for the given {@link WorkUnitState}, but if there are output data of * other tasks in the same folder, it may also publish those data. */ private void publishMultiTaskData(WorkUnitState state, int branchId, Set<Path> writerOutputPathsMoved) throws IOException { publishData(state, branchId, false, writerOutputPathsMoved); } protected void publishData(WorkUnitState state, int branchId, boolean publishSingleTaskData, Set<Path> writerOutputPathsMoved) throws IOException { // Get a ParallelRunner instance for moving files in parallel ParallelRunner parallelRunner = this.getParallelRunner(this.writerFileSystemByBranches.get(branchId)); // The directory where the workUnitState wrote its output data. Path writerOutputDir = WriterUtils.getWriterOutputDir(state, this.numBranches, branchId); if (!this.writerFileSystemByBranches.get(branchId).exists(writerOutputDir)) { LOG.warn(String.format("Branch %d of WorkUnit %s produced no data", branchId, state.getId())); return; } // The directory where the final output directory for this job will be placed. // It is a combination of DATA_PUBLISHER_FINAL_DIR and WRITER_FILE_PATH. Path publisherOutputDir = getPublisherOutputDir(state, branchId); if (publishSingleTaskData) { // Create final output directory WriterUtils.mkdirsWithRecursivePermission(this.publisherFileSystemByBranches.get(branchId), publisherOutputDir, this.permissions.get(branchId)); addSingleTaskWriterOutputToExistingDir(writerOutputDir, publisherOutputDir, state, branchId, parallelRunner); } else { if (writerOutputPathsMoved.contains(writerOutputDir)) { // This writer output path has already been moved for another task of the same extract // If publishSingleTaskData=true, writerOutputPathMoved is ignored. return; } if (this.publisherFileSystemByBranches.get(branchId).exists(publisherOutputDir)) { // The final output directory already exists, check if the job is configured to replace it. // If publishSingleTaskData=true, final output directory is never replaced. boolean replaceFinalOutputDir = this.getState() .getPropAsBoolean(ForkOperatorUtils.getPropertyNameForBranch( ConfigurationKeys.DATA_PUBLISHER_REPLACE_FINAL_DIR, this.numBranches, branchId)); // If the final output directory is not configured to be replaced, put new data to the existing directory. if (!replaceFinalOutputDir) { addWriterOutputToExistingDir(writerOutputDir, publisherOutputDir, state, branchId, parallelRunner); writerOutputPathsMoved.add(writerOutputDir); return; } // Delete the final output directory if it is configured to be replaced LOG.info("Deleting publisher output dir " + publisherOutputDir); this.publisherFileSystemByBranches.get(branchId).delete(publisherOutputDir, true); } else { // Create the parent directory of the final output directory if it does not exist WriterUtils.mkdirsWithRecursivePermission(this.publisherFileSystemByBranches.get(branchId), publisherOutputDir.getParent(), this.permissions.get(branchId)); } movePath(parallelRunner, state, writerOutputDir, publisherOutputDir, branchId); writerOutputPathsMoved.add(writerOutputDir); } } /** * Get the output directory path this {@link BaseDataPublisher} will write to. * * <p> * This is the default implementation. Subclasses of {@link BaseDataPublisher} may override this * to write to a custom directory or write using a custom directory structure or naming pattern. * </p> * * @param workUnitState a {@link WorkUnitState} object * @param branchId the fork branch ID * @return the output directory path this {@link BaseDataPublisher} will write to */ protected Path getPublisherOutputDir(WorkUnitState workUnitState, int branchId) { return WriterUtils.getDataPublisherFinalDir(workUnitState, this.numBranches, branchId); } protected void addSingleTaskWriterOutputToExistingDir(Path writerOutputDir, Path publisherOutputDir, WorkUnitState workUnitState, int branchId, ParallelRunner parallelRunner) throws IOException { String outputFilePropName = ForkOperatorUtils.getPropertyNameForBranch( ConfigurationKeys.WRITER_FINAL_OUTPUT_FILE_PATHS, this.numBranches, branchId); if (!workUnitState.contains(outputFilePropName)) { LOG.warn("Missing property " + outputFilePropName + ". This task may have pulled no data."); return; } Iterable<String> taskOutputFiles = workUnitState.getPropAsSet(outputFilePropName); for (String taskOutputFile : taskOutputFiles) { Path taskOutputPath = new Path(taskOutputFile); if (!this.writerFileSystemByBranches.get(branchId).exists(taskOutputPath)) { LOG.warn("Task output file " + taskOutputFile + " doesn't exist."); continue; } String pathSuffix = taskOutputFile.substring( taskOutputFile.indexOf(writerOutputDir.toString()) + writerOutputDir.toString().length() + 1); Path publisherOutputPath = new Path(publisherOutputDir, pathSuffix); WriterUtils.mkdirsWithRecursivePermission(this.publisherFileSystemByBranches.get(branchId), publisherOutputPath.getParent(), this.permissions.get(branchId)); movePath(parallelRunner, workUnitState, taskOutputPath, publisherOutputPath, branchId); } } protected void addWriterOutputToExistingDir(Path writerOutputDir, Path publisherOutputDir, WorkUnitState workUnitState, int branchId, ParallelRunner parallelRunner) throws IOException { boolean preserveFileName = workUnitState.getPropAsBoolean(ForkOperatorUtils.getPropertyNameForBranch( ConfigurationKeys.SOURCE_FILEBASED_PRESERVE_FILE_NAME, this.numBranches, branchId), false); // Go through each file in writerOutputDir and move it into publisherOutputDir for (FileStatus status : this.writerFileSystemByBranches.get(branchId).listStatus(writerOutputDir)) { // Preserve the file name if configured, use specified name otherwise Path finalOutputPath = preserveFileName ? new Path(publisherOutputDir, workUnitState.getProp(ForkOperatorUtils.getPropertyNameForBranch( ConfigurationKeys.DATA_PUBLISHER_FINAL_NAME, this.numBranches, branchId))) : new Path(publisherOutputDir, status.getPath().getName()); movePath(parallelRunner, workUnitState, status.getPath(), finalOutputPath, branchId); } } protected void movePath(ParallelRunner parallelRunner, State state, Path src, Path dst, int branchId) throws IOException { LOG.info(String.format("Moving %s to %s", src, dst)); boolean overwrite = state.getPropAsBoolean(ConfigurationKeys.DATA_PUBLISHER_OVERWRITE_ENABLED, false); this.publisherOutputDirs.addAll(recordPublisherOutputDirs(src, dst, branchId)); parallelRunner.movePath(src, this.publisherFileSystemByBranches.get(branchId), dst, overwrite, this.publisherFinalDirOwnerGroupsByBranches.get(branchId)); } protected Collection<Path> recordPublisherOutputDirs(Path src, Path dst, int branchId) throws IOException { // Getting file status from src rather than dst, because at this time dst doesn't yet exist. // If src is a dir, add dst to the set of paths. Otherwise, add dst's parent. if (this.writerFileSystemByBranches.get(branchId).getFileStatus(src).isDirectory()) { return ImmutableList.<Path>of(dst); } return ImmutableList.<Path>of(dst.getParent()); } private ParallelRunner getParallelRunner(FileSystem fs) { String uri = fs.getUri().toString(); if (!this.parallelRunners.containsKey(uri)) { this.parallelRunners.put(uri, this.parallelRunnerCloser.register(new ParallelRunner(this.parallelRunnerThreads, fs))); } return this.parallelRunners.get(uri); } /** * Merge all of the metadata output from each work-unit and publish the merged record. * @param states States from all tasks * @throws IOException If there is an error publishing the file */ @Override public void publishMetadata(Collection<? extends WorkUnitState> states) throws IOException { if (metadataMergers != null) { mergeAndPublishMetadata(states); } else { for (WorkUnitState workUnitState : states) { publishMetadata(workUnitState); } } } private void mergeAndPublishMetadata(Collection<? extends WorkUnitState> states) throws IOException { Set<String> partitions = new HashSet<>(); // There should be one merged metadata file per branch; first merge all of the pieces together mergeMetadataAndCollectPartitionNames(states, partitions); // Now, pick an arbitrary WorkUnitState and publish its metadata. We assume that publisher config settings // are the same across all workunits so it doesn't really matter which workUnit we call publishMetadata() for. WorkUnitState anyState = states.iterator().next(); for (int branchId = 0; branchId < numBranches; branchId++) { String mdOutputPath = getMetadataOutputPathFromState(anyState, branchId); List<Path> metadataPaths = Lists.newArrayList(); String userSpecifiedPath = getUserSpecifiedOutputPathFromState(anyState, branchId); if (partitions.isEmpty() || userSpecifiedPath != null) { metadataPaths.add(getMetadataOutputFileForBranch(anyState, branchId)); } else { for (String partition : partitions) { metadataPaths.add(new Path(new Path(mdOutputPath, partition), getMetadataFileNameForBranch(anyState, branchId))); } } publishMetadata(getMergedMetadataForState(anyState, branchId), branchId, metadataPaths); } } private void mergeMetadataAndCollectPartitionNames(Collection<? extends WorkUnitState> states, Set<String> partitions) { for (WorkUnitState workUnitState : states) { for (int branchId = 0; branchId < numBranches; branchId++) { String md = getIntermediateMetadataFromState(workUnitState, branchId); if (md != null) { metadataMergers.get(branchId).update(md); } } for (Map.Entry<Object, Object> property : workUnitState.getProperties().entrySet()) { if (((String) property.getKey()).startsWith(ConfigurationKeys.WRITER_PARTITION_PATH_KEY)) { partitions.add((String) property.getValue()); } } } } /** * Publish metadata for each branch. We expect the metadata to be of String format and * populated in either the WRITER_MERGED_METADATA_KEY state or the WRITER_METADATA_KEY configuration key. */ @Override public void publishMetadata(WorkUnitState state) throws IOException { // For each branch - retrieve metadata and metadata output directory from task state; then publish for (int branchId = 0; branchId < this.numBranches; branchId++) { String metadataValue = getMergedMetadataForState(state, branchId); if (metadataValue == null) { //Nothing to write continue; } Path metadataOutputPath = getMetadataOutputFileForBranch(state, branchId); publishMetadata(metadataValue, branchId, Collections.singleton(metadataOutputPath)); } } /** * Publish metadata to a set of paths */ private void publishMetadata(String metadataValue, int branchId, Collection<Path> pathsToPublish) throws IOException { for (Path metadataOutputPath : pathsToPublish) { try { if (metadataOutputPath == null) { LOG.info("Metadata output path not set for branch " + String.valueOf(branchId) + ", not publishing."); continue; } FileSystem fs = this.metaDataWriterFileSystemByBranches.get(branchId); if (!fs.exists(metadataOutputPath.getParent())) { WriterUtils.mkdirsWithRecursivePermission(fs, metadataOutputPath, this.permissions.get(branchId)); } //Delete the file if metadata already exists if (fs.exists(metadataOutputPath)) { HadoopUtils.deletePath(fs, metadataOutputPath, false); } LOG.info("Writing metadata for branch " + String.valueOf(branchId) + " to " + metadataOutputPath.toString()); try (FSDataOutputStream outputStream = fs.create(metadataOutputPath)) { outputStream.write(metadataValue.getBytes(StandardCharsets.UTF_8)); } } catch (IOException e) { LOG.error("Metadata file is not generated: " + e, e); } } } private String getMetadataFileNameForBranch(WorkUnitState state, int branchId) { // Note: This doesn't follow the pattern elsewhere in Gobblin where we have branch specific config // parameters! Leaving this way for backwards compatibility. String filePrefix = state.getProp(ConfigurationKeys.DATA_PUBLISHER_METADATA_OUTPUT_FILE); return ForkOperatorUtils.getPropertyNameForBranch(filePrefix, this.numBranches, branchId); } private Path getMetadataOutputFileForBranch(WorkUnitState state, int branchId) { String metaDataOutputDirStr = getMetadataOutputPathFromState(state, branchId); String fileName = getMetadataFileNameForBranch(state, branchId); if (metaDataOutputDirStr == null || fileName == null) { return null; } return new Path(metaDataOutputDirStr, fileName); } private String getUserSpecifiedOutputPathFromState(WorkUnitState state, int branchId) { String outputDir = state.getProp(ForkOperatorUtils.getPropertyNameForBranch( ConfigurationKeys.DATA_PUBLISHER_METADATA_OUTPUT_DIR, this.numBranches, branchId)); // An older version of this code did not get a branch specific PUBLISHER_METADATA_OUTPUT_DIR so fallback // for compatibility's sake if (outputDir == null && this.numBranches > 1) { outputDir = state.getProp(ConfigurationKeys.DATA_PUBLISHER_METADATA_OUTPUT_DIR); if (outputDir != null) { LOG.warn( "Branches are configured for this job but a per branch metadata output directory was not set;" + " is this intended?"); } } return outputDir; } private String getMetadataOutputPathFromState(WorkUnitState state, int branchId) { String outputDir = getUserSpecifiedOutputPathFromState(state, branchId); // Just write out to the regular output path if a metadata specific path hasn't been provided if (outputDir == null) { String publisherOutputDir = getPublisherOutputDir(state, branchId).toString(); LOG.info("Missing metadata output directory path : " + ConfigurationKeys.DATA_PUBLISHER_METADATA_OUTPUT_DIR + " in the config; assuming outputPath " + publisherOutputDir); return publisherOutputDir; } return outputDir; } /* * Retrieve intermediate metadata (eg the metadata stored by each writer) for a given state and branch id. */ private String getIntermediateMetadataFromState(WorkUnitState state, int branchId) { return state.getProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_METADATA_KEY, this.numBranches, branchId)); } /* * Get the merged metadata given a workunit state and branch id. This method assumes * all intermediate metadata has already been passed to the MetadataMerger. * * If metadata mergers are not configured, instead return the metadata from job config that was * passed in by the user. */ private String getMergedMetadataForState(WorkUnitState state, int branchId) { String mergedMd = null; if (metadataMergers != null && metadataMergers.get(branchId) != null) { mergedMd = metadataMergers.get(branchId).getMergedMetadata(); if (mergedMd == null) { LOG.warn("Metadata merger for branch {} returned null - bug in merger?", branchId); } } if (mergedMd == null) { mergedMd = state.getProp(ConfigurationKeys.DATA_PUBLISHER_METADATA_STR); } return mergedMd; } private boolean shouldPublishMetadataForBranch(int branchId) { String keyName = ForkOperatorUtils.getPropertyNameForBranch( ConfigurationKeys.DATA_PUBLISH_WRITER_METADATA_KEY, this.numBranches, branchId); return this.getState().getPropAsBoolean(keyName, false); } /** * The BaseDataPublisher relies on publishData() to create and clean-up the output directories, so data * has to be published before the metadata can be. */ @Override protected boolean shouldPublishMetadataFirst() { return false; } }