Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package gobblin.data.management.copy.hive; import java.io.IOException; import java.net.URI; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InvalidInputException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.thrift.TException; import com.google.common.annotations.VisibleForTesting; import com.typesafe.config.ConfigFactory; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.io.Closer; import com.google.gson.Gson; import com.typesafe.config.Config; import gobblin.commit.CommitStep; import gobblin.configuration.State; import gobblin.util.ClassAliasResolver; import gobblin.data.management.copy.CopyConfiguration; import gobblin.data.management.copy.CopyEntity; import gobblin.data.management.copy.CopyableFile; import gobblin.data.management.copy.OwnerAndPermission; import gobblin.data.management.copy.entities.PostPublishStep; import gobblin.data.management.copy.hive.avro.HiveAvroCopyEntityHelper; import gobblin.data.management.partition.FileSet; import gobblin.hive.HiveMetastoreClientPool; import gobblin.hive.HiveRegProps; import gobblin.hive.HiveRegisterStep; import gobblin.hive.PartitionDeregisterStep; import gobblin.hive.TableDeregisterStep; import gobblin.hive.metastore.HiveMetaStoreUtils; import gobblin.hive.spec.HiveSpec; import gobblin.hive.spec.SimpleHiveSpec; import gobblin.metrics.event.EventSubmitter; import gobblin.metrics.event.MultiTimingEvent; import gobblin.util.PathUtils; import gobblin.util.commit.DeleteFileCommitStep; import gobblin.util.reflection.GobblinConstructorUtils; import gobblin.util.request_allocation.PushDownRequestor; import lombok.Builder; import lombok.Data; import lombok.Getter; import lombok.Singular; import lombok.ToString; import lombok.extern.slf4j.Slf4j; /** * Creates {@link CopyEntity}s for copying a Hive table. */ @Slf4j @Getter public class HiveCopyEntityHelper { public static final String EXISTING_ENTITY_POLICY_KEY = HiveDatasetFinder.HIVE_DATASET_PREFIX + ".existing.entity.conflict.policy"; public static final String DEFAULT_EXISTING_ENTITY_POLICY = ExistingEntityPolicy.ABORT.name(); public static final String UNMANAGED_DATA_POLICY_KEY = HiveDatasetFinder.HIVE_DATASET_PREFIX + ".unmanaged.data.conflict.policy"; public static final String DEFAULT_UNMANAGED_DATA_POLICY = UnmanagedDataPolicy.ABORT.name(); /** Target metastore URI */ public static final String TARGET_METASTORE_URI_KEY = HiveDatasetFinder.HIVE_DATASET_PREFIX + ".copy.target.metastore.uri"; /** Target database name */ public static final String TARGET_DATABASE_KEY = HiveDatasetFinder.HIVE_DATASET_PREFIX + ".copy.target.database"; /** A filter to select partitions to copy */ public static final String COPY_PARTITIONS_FILTER_CONSTANT = HiveDatasetFinder.HIVE_DATASET_PREFIX + ".copy.partition.filter.constant"; /** Use an implementation of {@link PartitionFilterGenerator} to dynamically create partition filter. The value should * be the name of the implementation to use. */ public static final String COPY_PARTITION_FILTER_GENERATOR = HiveDatasetFinder.HIVE_DATASET_PREFIX + ".copy.partition.filter.generator"; /** A predicate applied to each partition before any file listing. * If the predicate returns true, the partition will be skipped. */ public static final String FAST_PARTITION_SKIP_PREDICATE = HiveDatasetFinder.HIVE_DATASET_PREFIX + ".copy.fast.partition.skip.predicate"; /** A predicate applied to non partition table before any file listing. * If the predicate returns true, the table will be skipped. */ public static final String FAST_TABLE_SKIP_PREDICATE = HiveDatasetFinder.HIVE_DATASET_PREFIX + ".copy.fast.table.skip.predicate"; /** Method for deleting files on deregister. One of {@link DeregisterFileDeleteMethod}. */ public static final String DELETE_FILES_ON_DEREGISTER = HiveDatasetFinder.HIVE_DATASET_PREFIX + ".copy.deregister.fileDeleteMethod"; public static final DeregisterFileDeleteMethod DEFAULT_DEREGISTER_DELETE_METHOD = DeregisterFileDeleteMethod.NO_DELETE; /** * Config key to specify if {@link IMetaStoreClient }'s filtering method {@link listPartitionsByFilter} is not enough * for filtering out specific partitions. * For example, if you specify "Path" as the filter type and "Hourly" as the filtering condition, * partitions with Path containing '/Hourly/' will be kept. */ public static final String HIVE_PARTITION_EXTENDED_FILTER_TYPE = HiveDatasetFinder.HIVE_DATASET_PREFIX + ".extendedFilterType"; static final Gson gson = new Gson(); private static final String source_client = "source_client"; private static final String target_client = "target_client"; public static final String GOBBLIN_DISTCP = "gobblin-distcp"; public static class Stages { public static final String EXISTING_PARTITION = "ExistingPartition"; public static final String PARTITION_SKIP_PREDICATE = "PartitionSkipPredicate"; public static final String CREATE_LOCATIONS = "CreateLocations"; public static final String FULL_PATH_DIFF = "FullPathDiff"; public static final String CREATE_DELETE_UNITS = "CreateDeleteUnits"; public static final String CREATE_COPY_UNITS = "CreateCopyUnits"; public static final String SOURCE_PATH_LISTING = "SourcePathListing"; public static final String TARGET_EXISTING_PATH_LISTING = "TargetExistingPathListing"; public static final String DESIRED_PATHS_LISTING = "DesiredPathsListing"; public static final String PATH_DIFF = "PathDiff"; public static final String COMPUTE_DELETE_PATHS = "ComputeDeletePaths"; public static final String GET_TABLES = "GetTables"; public static final String COMPUTE_TARGETS = "ComputeTargets"; } private final long startTime; private final HiveDataset dataset; private final CopyConfiguration configuration; private final FileSystem targetFs; private final HiveMetastoreClientPool targetClientPool; private final String targetDatabase; private final HiveRegProps hiveRegProps; private Optional<Table> existingTargetTable; private final Table targetTable; private final Optional<String> targetURI; private final ExistingEntityPolicy existingEntityPolicy; private final UnmanagedDataPolicy unmanagedDataPolicy; private final Optional<String> partitionFilter; private Optional<? extends HivePartitionExtendedFilter> hivePartitionExtendedFilter; private final Optional<Predicate<HivePartitionFileSet>> fastPartitionSkip; private final Optional<Predicate<HiveCopyEntityHelper>> fastTableSkip; private final DeregisterFileDeleteMethod deleteMethod; private final Optional<CommitStep> tableRegistrationStep; private final Map<List<String>, Partition> sourcePartitions; private final Map<List<String>, Partition> targetPartitions; private final EventSubmitter eventSubmitter; @Getter protected final HiveTargetPathHelper targetPathHelper; /** * Defines what should be done for partitions that exist in the target but are not compatible with the source. */ public enum ExistingEntityPolicy { /** Deregister target partition, delete its files, and create a new partition with correct values. */ REPLACE_PARTITIONS, /** Deregister target table, do NOT delete its files, and create a new table with correct values. */ REPLACE_TABLE, /** Keep the target table as registered while updating the file location */ UPDATE_TABLE, /** Abort copying of conflict table. */ ABORT } /** * Defines what should be done for data that is not managed by the existing target table / partition. */ public enum UnmanagedDataPolicy { /** Delete any data that is not managed by the existing target table / partition. */ DELETE_UNMANAGED_DATA, /** Abort copying of conflict table / partition. */ ABORT } public enum DeregisterFileDeleteMethod { /** Delete the files pointed at by the input format. */ INPUT_FORMAT, /** Delete all files at the partition location recursively. */ RECURSIVE, /** Don't delete files, just deregister partition. */ NO_DELETE } /** * A container for the differences between desired and existing files. */ @Builder @ToString protected static class DiffPathSet { /** Desired files that don't exist on target */ @Singular(value = "copyFile") Collection<FileStatus> filesToCopy; /** Files in target that are not desired */ @Singular(value = "deleteFile") Collection<Path> pathsToDelete; } /** * Represents a source {@link FileStatus} and a {@link Path} destination. */ @Data private static class SourceAndDestination { private final FileStatus source; private final Path destination; } HiveCopyEntityHelper(HiveDataset dataset, CopyConfiguration configuration, FileSystem targetFs) throws IOException { try (Closer closer = Closer.create()) { log.info("Finding copy entities for table " + dataset.table.getCompleteName()); this.eventSubmitter = new EventSubmitter.Builder(dataset.getMetricContext(), "hive.dataset.copy") .build(); MultiTimingEvent multiTimer = closer .register(new MultiTimingEvent(this.eventSubmitter, "HiveCopySetup", true)); this.startTime = System.currentTimeMillis(); this.dataset = dataset; this.configuration = configuration; this.targetFs = targetFs; this.targetPathHelper = new HiveTargetPathHelper(this.dataset); this.hiveRegProps = new HiveRegProps(new State(this.dataset.getProperties())); this.targetURI = Optional .fromNullable(this.dataset.getProperties().getProperty(TARGET_METASTORE_URI_KEY)); this.targetClientPool = HiveMetastoreClientPool.get(this.dataset.getProperties(), this.targetURI); this.targetDatabase = Optional .fromNullable(this.dataset.getProperties().getProperty(TARGET_DATABASE_KEY)) .or(this.dataset.table.getDbName()); this.existingEntityPolicy = ExistingEntityPolicy.valueOf(this.dataset.getProperties() .getProperty(EXISTING_ENTITY_POLICY_KEY, DEFAULT_EXISTING_ENTITY_POLICY).toUpperCase()); this.unmanagedDataPolicy = UnmanagedDataPolicy.valueOf(this.dataset.getProperties() .getProperty(UNMANAGED_DATA_POLICY_KEY, DEFAULT_UNMANAGED_DATA_POLICY).toUpperCase()); this.deleteMethod = this.dataset.getProperties().containsKey(DELETE_FILES_ON_DEREGISTER) ? DeregisterFileDeleteMethod.valueOf( this.dataset.getProperties().getProperty(DELETE_FILES_ON_DEREGISTER).toUpperCase()) : DEFAULT_DEREGISTER_DELETE_METHOD; if (this.dataset.getProperties().containsKey(COPY_PARTITION_FILTER_GENERATOR)) { try { PartitionFilterGenerator generator = GobblinConstructorUtils.invokeFirstConstructor( (Class<PartitionFilterGenerator>) Class.forName( this.dataset.getProperties().getProperty(COPY_PARTITION_FILTER_GENERATOR)), Lists.<Object>newArrayList(this.dataset.getProperties()), Lists.newArrayList()); this.partitionFilter = Optional.of(generator.getFilter(this.dataset)); log.info(String.format("Dynamic partition filter for table %s: %s.", this.dataset.table.getCompleteName(), this.partitionFilter.get())); } catch (ReflectiveOperationException roe) { throw new IOException(roe); } } else { this.partitionFilter = Optional .fromNullable(this.dataset.getProperties().getProperty(COPY_PARTITIONS_FILTER_CONSTANT)); } // Initialize extended partition filter if (this.dataset.getProperties().containsKey(HIVE_PARTITION_EXTENDED_FILTER_TYPE)) { String filterType = dataset.getProperties().getProperty(HIVE_PARTITION_EXTENDED_FILTER_TYPE); try { Config config = ConfigFactory.parseProperties(this.dataset.getProperties()); this.hivePartitionExtendedFilter = Optional .of(new ClassAliasResolver<>(HivePartitionExtendedFilterFactory.class) .resolveClass(filterType).newInstance().createFilter(config)); } catch (ReflectiveOperationException roe) { log.error("Error: Could not find filter with alias " + filterType); closer.close(); throw new IOException(roe); } } else { this.hivePartitionExtendedFilter = Optional.absent(); } try { this.fastPartitionSkip = this.dataset.getProperties().containsKey(FAST_PARTITION_SKIP_PREDICATE) ? Optional.of(GobblinConstructorUtils.invokeFirstConstructor( (Class<Predicate<HivePartitionFileSet>>) Class.forName( this.dataset.getProperties().getProperty(FAST_PARTITION_SKIP_PREDICATE)), Lists.<Object>newArrayList(this), Lists.newArrayList())) : Optional.<Predicate<HivePartitionFileSet>>absent(); this.fastTableSkip = this.dataset.getProperties().containsKey(FAST_TABLE_SKIP_PREDICATE) ? Optional.of(GobblinConstructorUtils.invokeFirstConstructor( (Class<Predicate<HiveCopyEntityHelper>>) Class.forName( this.dataset.getProperties().getProperty(FAST_TABLE_SKIP_PREDICATE)), Lists.newArrayList())) : Optional.<Predicate<HiveCopyEntityHelper>>absent(); } catch (ReflectiveOperationException roe) { closer.close(); throw new IOException(roe); } Map<String, HiveMetastoreClientPool> namedPools = ImmutableMap.of(source_client, this.dataset.clientPool, target_client, this.targetClientPool); multiTimer.nextStage(Stages.GET_TABLES); try (HiveMetastoreClientPool.MultiClient multiClient = HiveMetastoreClientPool .safeGetClients(namedPools)) { if (multiClient.getClient(target_client).tableExists(this.targetDatabase, this.dataset.table.getTableName())) { this.existingTargetTable = Optional.of(new Table(multiClient.getClient(target_client) .getTable(this.targetDatabase, this.dataset.table.getTableName()))); } else { this.existingTargetTable = Optional.absent(); } // Constructing CommitStep object for table registration Path targetPath = getTargetLocation(dataset.fs, this.targetFs, dataset.table.getDataLocation(), Optional.<Partition>absent()); this.targetTable = getTargetTable(this.dataset.table, targetPath); HiveSpec tableHiveSpec = new SimpleHiveSpec.Builder<>(targetPath) .withTable(HiveMetaStoreUtils.getHiveTable(this.targetTable.getTTable())).build(); CommitStep tableRegistrationStep = new HiveRegisterStep(this.targetURI, tableHiveSpec, this.hiveRegProps); this.tableRegistrationStep = Optional.of(tableRegistrationStep); if (this.existingTargetTable.isPresent() && this.existingTargetTable.get().isPartitioned()) { checkPartitionedTableCompatibility(this.targetTable, this.existingTargetTable.get()); } if (HiveUtils.isPartitioned(this.dataset.table)) { this.sourcePartitions = HiveUtils.getPartitionsMap(multiClient.getClient(source_client), this.dataset.table, this.partitionFilter, this.hivePartitionExtendedFilter); // Note: this must be mutable, so we copy the map this.targetPartitions = this.existingTargetTable.isPresent() ? Maps.newHashMap(HiveUtils.getPartitionsMap(multiClient.getClient(target_client), this.existingTargetTable.get(), this.partitionFilter, this.hivePartitionExtendedFilter)) : Maps.<List<String>, Partition>newHashMap(); } else { this.sourcePartitions = Maps.newHashMap(); this.targetPartitions = Maps.newHashMap(); } } catch (TException te) { closer.close(); throw new IOException("Failed to generate work units for table " + dataset.table.getCompleteName(), te); } } } /** * See {@link #getCopyEntities(CopyConfiguration, Comparator, PushDownRequestor)}. This method does not pushdown any prioritizer. */ Iterator<FileSet<CopyEntity>> getCopyEntities(CopyConfiguration configuration) throws IOException { return getCopyEntities(configuration, null, null); } /** * Finds all files read by the table and generates {@link CopyEntity}s for duplicating the table. The semantics are as follows: * 1. Find all valid {@link org.apache.hadoop.hive.metastore.api.StorageDescriptor}. If the table is partitioned, the * {@link org.apache.hadoop.hive.metastore.api.StorageDescriptor} of the base * table will be ignored, and we will instead process the {@link org.apache.hadoop.hive.metastore.api.StorageDescriptor} of each partition. * 2. For each {@link org.apache.hadoop.hive.metastore.api.StorageDescriptor} find all files referred by it. * 3. Generate a {@link CopyableFile} for each file referred by a {@link org.apache.hadoop.hive.metastore.api.StorageDescriptor}. * 4. If the table is partitioned, create a file set for each partition. * 5. Create work units for registering, deregistering partitions / tables, and deleting unnecessary files in the target. * * For computation of target locations see {@link HiveTargetPathHelper#getTargetPath} */ Iterator<FileSet<CopyEntity>> getCopyEntities(CopyConfiguration configuration, Comparator<FileSet<CopyEntity>> prioritizer, PushDownRequestor<FileSet<CopyEntity>> requestor) throws IOException { if (HiveUtils.isPartitioned(this.dataset.table)) { return new PartitionIterator(this.sourcePartitions, configuration, prioritizer, requestor); } else { FileSet<CopyEntity> fileSet = new UnpartitionedTableFileSet(this.dataset.table.getCompleteName(), this.dataset, this); return Iterators.singletonIterator(fileSet); } } /** * An iterator producing a {@link FileSet} of {@link CopyEntity} for each partition in this table. The files * are not scanned or the {@link FileSet} materialized until {@link #next} is called. */ private class PartitionIterator implements Iterator<FileSet<CopyEntity>> { static final String DEREGISTER_FILE_SET = "deregister"; private final List<FileSet<CopyEntity>> allFileSets; private final Iterator<FileSet<CopyEntity>> fileSetIterator; public PartitionIterator(Map<List<String>, Partition> partitionMap, CopyConfiguration configuration, Comparator<FileSet<CopyEntity>> prioritizer, PushDownRequestor<FileSet<CopyEntity>> requestor) { this.allFileSets = generateAllFileSets(partitionMap); for (FileSet<CopyEntity> fileSet : this.allFileSets) { fileSet.setRequestor(requestor); } if (prioritizer != null) { Collections.sort(this.allFileSets, prioritizer); } this.fileSetIterator = this.allFileSets.iterator(); } @Override public boolean hasNext() { return this.fileSetIterator.hasNext(); } @Override public FileSet<CopyEntity> next() { return this.fileSetIterator.next(); } @Override public void remove() { throw new UnsupportedOperationException(); } private List<FileSet<CopyEntity>> generateAllFileSets(Map<List<String>, Partition> partitionMap) { List<FileSet<CopyEntity>> fileSets = Lists.newArrayList(); for (Map.Entry<List<String>, Partition> partition : partitionMap.entrySet()) { fileSets.add(fileSetForPartition(partition.getValue())); HiveCopyEntityHelper.this.targetPartitions.remove(partition.getKey()); } if (!HiveCopyEntityHelper.this.targetPartitions.isEmpty()) { fileSets.add( new HivePartitionsDeregisterFileSet( HiveCopyEntityHelper.this.dataset.getTable().getCompleteName() + DEREGISTER_FILE_SET, HiveCopyEntityHelper.this.dataset, HiveCopyEntityHelper.this.targetPartitions.values(), HiveCopyEntityHelper.this)); } return fileSets; } private FileSet<CopyEntity> fileSetForPartition(final Partition partition) { return new HivePartitionFileSet(HiveCopyEntityHelper.this, partition, HiveCopyEntityHelper.this.dataset.getProperties()); } } private Table getTargetTable(Table originTable, Path targetLocation) throws IOException { try { Table targetTable = originTable.copy(); targetTable.setDbName(this.targetDatabase); targetTable.setDataLocation(targetLocation); /* * Need to set the table owner as the flow executor */ targetTable.setOwner(UserGroupInformation.getCurrentUser().getShortUserName()); targetTable.getTTable().putToParameters(HiveDataset.REGISTERER, GOBBLIN_DISTCP); targetTable.getTTable().putToParameters(HiveDataset.REGISTRATION_GENERATION_TIME_MILLIS, Long.toString(this.startTime)); targetTable.getTTable().unsetCreateTime(); HiveAvroCopyEntityHelper.updateTableAttributesIfAvro(targetTable, this); return targetTable; } catch (HiveException he) { throw new IOException(he); } } int addPartitionDeregisterSteps(List<CopyEntity> copyEntities, String fileSet, int initialPriority, Table table, Partition partition) throws IOException { int stepPriority = initialPriority; Collection<Path> partitionPaths = Lists.newArrayList(); if (this.deleteMethod == DeregisterFileDeleteMethod.RECURSIVE) { partitionPaths = Lists.newArrayList(partition.getDataLocation()); } else if (this.deleteMethod == DeregisterFileDeleteMethod.INPUT_FORMAT) { InputFormat<?, ?> inputFormat = HiveUtils.getInputFormat(partition.getTPartition().getSd()); HiveLocationDescriptor targetLocation = new HiveLocationDescriptor(partition.getDataLocation(), inputFormat, this.targetFs, this.dataset.getProperties()); partitionPaths = targetLocation.getPaths().keySet(); } else if (this.deleteMethod == DeregisterFileDeleteMethod.NO_DELETE) { partitionPaths = Lists.newArrayList(); } if (!partitionPaths.isEmpty()) { DeleteFileCommitStep deletePaths = DeleteFileCommitStep.fromPaths(this.targetFs, partitionPaths, this.dataset.getProperties(), table.getDataLocation()); copyEntities.add( new PostPublishStep(fileSet, Maps.<String, String>newHashMap(), deletePaths, stepPriority++)); } PartitionDeregisterStep deregister = new PartitionDeregisterStep(table.getTTable(), partition.getTPartition(), this.targetURI, this.hiveRegProps); copyEntities .add(new PostPublishStep(fileSet, Maps.<String, String>newHashMap(), deregister, stepPriority++)); return stepPriority; } @VisibleForTesting protected int addTableDeregisterSteps(List<CopyEntity> copyEntities, String fileSet, int initialPriority, Table table) throws IOException { int stepPriority = initialPriority; Collection<Path> tablePaths = Lists.newArrayList(); switch (this.getDeleteMethod()) { case RECURSIVE: tablePaths = Lists.newArrayList(table.getDataLocation()); break; case INPUT_FORMAT: InputFormat<?, ?> inputFormat = HiveUtils.getInputFormat(table.getSd()); HiveLocationDescriptor targetLocation = new HiveLocationDescriptor(table.getDataLocation(), inputFormat, this.getTargetFs(), this.getDataset().getProperties()); tablePaths = targetLocation.getPaths().keySet(); break; case NO_DELETE: tablePaths = Lists.newArrayList(); break; default: tablePaths = Lists.newArrayList(); } if (!tablePaths.isEmpty()) { DeleteFileCommitStep deletePaths = DeleteFileCommitStep.fromPaths(this.getTargetFs(), tablePaths, this.getDataset().getProperties(), table.getDataLocation()); copyEntities.add( new PostPublishStep(fileSet, Maps.<String, String>newHashMap(), deletePaths, stepPriority++)); } TableDeregisterStep deregister = new TableDeregisterStep(table.getTTable(), this.getTargetURI(), this.getHiveRegProps()); copyEntities .add(new PostPublishStep(fileSet, Maps.<String, String>newHashMap(), deregister, stepPriority++)); return stepPriority; } int addSharedSteps(List<CopyEntity> copyEntities, String fileSet, int initialPriority) { int priority = initialPriority; if (this.tableRegistrationStep.isPresent()) { copyEntities.add(new PostPublishStep(fileSet, Maps.<String, String>newHashMap(), this.tableRegistrationStep.get(), priority++)); } return priority; } /** * Compares three entities to figure out which files should be copied and which files should be deleted in the target * file system. * @param sourceLocation Represents the source table or partition. * @param desiredTargetLocation Represents the new desired table or partition. * @param currentTargetLocation Represents the corresponding existing table or partition in the target hcat if it exists. * @param partition If present, contains partition information. * @return A {@link DiffPathSet} with data on files to copy and delete. * @throws IOException if the copy of this table / partition should be aborted. */ @VisibleForTesting protected static DiffPathSet fullPathDiff(HiveLocationDescriptor sourceLocation, HiveLocationDescriptor desiredTargetLocation, Optional<HiveLocationDescriptor> currentTargetLocation, Optional<Partition> partition, MultiTimingEvent multiTimer, HiveCopyEntityHelper helper) throws IOException { DiffPathSet.DiffPathSetBuilder builder = DiffPathSet.builder(); multiTimer.nextStage(Stages.SOURCE_PATH_LISTING); // These are the paths at the source Map<Path, FileStatus> sourcePaths = sourceLocation.getPaths(); multiTimer.nextStage(Stages.TARGET_EXISTING_PATH_LISTING); // These are the paths that the existing target table / partition uses now Map<Path, FileStatus> targetExistingPaths = currentTargetLocation.isPresent() ? currentTargetLocation.get().getPaths() : Maps.<Path, FileStatus>newHashMap(); multiTimer.nextStage(Stages.DESIRED_PATHS_LISTING); // These are the paths that exist at the destination and the new table / partition would pick up Map<Path, FileStatus> desiredTargetExistingPaths; try { desiredTargetExistingPaths = desiredTargetLocation.getPaths(); } catch (InvalidInputException ioe) { // Thrown if inputFormat cannot find location in target. Since location doesn't exist, this set is empty. desiredTargetExistingPaths = Maps.newHashMap(); } multiTimer.nextStage(Stages.PATH_DIFF); for (FileStatus sourcePath : sourcePaths.values()) { // For each source path Path newPath = helper.getTargetPathHelper().getTargetPath(sourcePath.getPath(), desiredTargetLocation.getFileSystem(), partition, true); boolean shouldCopy = true; if (desiredTargetExistingPaths.containsKey(newPath)) { // If the file exists at the destination, check whether it should be replaced, if not, no need to copy FileStatus existingTargetStatus = desiredTargetExistingPaths.get(newPath); if (!helper.shouldReplaceFile(existingTargetStatus, sourcePath)) { shouldCopy = false; } } if (shouldCopy) { builder.copyFile(sourcePath); } else { // if not copying, we want to keep the file in the target // at the end of this loop, all files in targetExistingPaths will be marked for deletion, so remove this file targetExistingPaths.remove(newPath); desiredTargetExistingPaths.remove(newPath); } } multiTimer.nextStage(Stages.COMPUTE_DELETE_PATHS); // At this point, targetExistingPaths contains paths managed by target partition / table, but that we don't want // delete them for (Path delete : targetExistingPaths.keySet()) { builder.deleteFile(delete); desiredTargetExistingPaths.remove(delete); } // Now desiredTargetExistingPaths contains paths that we don't want, but which are not managed by the existing // table / partition. // Ideally, we shouldn't delete them (they're not managed by Hive), and we don't want to pick // them up in the new table / partition, so if there are any leftover files, we should abort copying // this table / partition. if (desiredTargetExistingPaths.size() > 0 && helper.getUnmanagedDataPolicy() != UnmanagedDataPolicy.DELETE_UNMANAGED_DATA) { throw new IOException(String.format( "New table / partition would pick up existing, undesired files in target file system. " + "%s, files %s.", partition.isPresent() ? partition.get().getCompleteName() : helper.getDataset().getTable().getCompleteName(), Arrays.toString(desiredTargetExistingPaths.keySet().toArray()))); } // Unless, the policy requires us to delete such un-managed files - in which case: we will add the leftover files // to the deletion list. else if (desiredTargetExistingPaths.size() > 0) { for (Path delete : desiredTargetExistingPaths.keySet()) { builder.deleteFile(delete); } log.warn(String.format( "Un-managed files detected in target file system, however deleting them " + "because of the policy: %s Files to be deleted are: %s", UnmanagedDataPolicy.DELETE_UNMANAGED_DATA, StringUtils.join(desiredTargetExistingPaths.keySet(), ","))); } return builder.build(); } private static boolean shouldReplaceFile(FileStatus referencePath, FileStatus replacementFile) { return replacementFile.getLen() != referencePath.getLen() || referencePath.getModificationTime() < replacementFile.getModificationTime(); } private void checkPartitionedTableCompatibility(Table desiredTargetTable, Table existingTargetTable) throws IOException { if (!desiredTargetTable.getDataLocation().equals(existingTargetTable.getDataLocation())) { throw new HiveTableLocationNotMatchException(desiredTargetTable.getDataLocation(), existingTargetTable.getDataLocation()); } if (HiveUtils.isPartitioned(desiredTargetTable) != HiveUtils.isPartitioned(existingTargetTable)) { throw new IOException(String.format( "%s: Desired target table %s partitioned, existing target table %s partitioned. Tables are incompatible.", this.dataset.tableIdentifier, HiveUtils.isPartitioned(desiredTargetTable) ? "is" : "is not", HiveUtils.isPartitioned(existingTargetTable) ? "is" : "is not")); } if (desiredTargetTable.isPartitioned() && !desiredTargetTable.getPartitionKeys().equals(existingTargetTable.getPartitionKeys())) { throw new IOException(String.format( "%s: Desired target table has partition keys %s, existing target table has partition keys %s. " + "Tables are incompatible.", this.dataset.tableIdentifier, gson.toJson(desiredTargetTable.getPartitionKeys()), gson.toJson(existingTargetTable.getPartitionKeys()))); } } /** * Get builders for a {@link CopyableFile} for each file referred to by a {@link org.apache.hadoop.hive.metastore.api.StorageDescriptor}. */ List<CopyableFile.Builder> getCopyableFilesFromPaths(Collection<FileStatus> paths, CopyConfiguration configuration, Optional<Partition> partition) throws IOException { List<CopyableFile.Builder> builders = Lists.newArrayList(); List<SourceAndDestination> dataFiles = Lists.newArrayList(); Configuration hadoopConfiguration = new Configuration(); FileSystem actualSourceFs = null; String referenceScheme = null; String referenceAuthority = null; for (FileStatus status : paths) { dataFiles.add(new SourceAndDestination(status, getTargetPathHelper().getTargetPath(status.getPath(), this.targetFs, partition, true))); } for (SourceAndDestination sourceAndDestination : dataFiles) { URI uri = sourceAndDestination.getSource().getPath().toUri(); if (actualSourceFs == null || !StringUtils.equals(referenceScheme, uri.getScheme()) || !StringUtils.equals(referenceAuthority, uri.getAuthority())) { actualSourceFs = sourceAndDestination.getSource().getPath().getFileSystem(hadoopConfiguration); referenceScheme = uri.getScheme(); referenceAuthority = uri.getAuthority(); } if (!this.dataset.getTableRootPath().isPresent()) { // The logic for computing ancestor owner and permissions for hive copies depends on tables having a non-glob // location. Currently, this restriction is also imposed by Hive, so this is not a problem. If this ever changes // on the Hive side, and we try to copy a table with a glob location, this logic will have to change. throw new IOException(String.format("Table %s does not have a concrete table root path.", this.dataset.getTable().getCompleteName())); } List<OwnerAndPermission> ancestorOwnerAndPermission = CopyableFile .resolveReplicatedOwnerAndPermissionsRecursively(actualSourceFs, sourceAndDestination.getSource().getPath().getParent(), this.dataset.getTableRootPath().get().getParent(), configuration); builders.add(CopyableFile .fromOriginAndDestination(actualSourceFs, sourceAndDestination.getSource(), sourceAndDestination.getDestination(), configuration) .ancestorsOwnerAndPermission(ancestorOwnerAndPermission)); } return builders; } /** * Compute the target location for a Hive location. * @param sourceFs Source {@link FileSystem}. * @param path source {@link Path} in Hive location. * @param partition partition these paths correspond to. * @return transformed location in the target. * @throws IOException if cannot generate a single target location. */ Path getTargetLocation(FileSystem sourceFs, FileSystem targetFs, Path path, Optional<Partition> partition) throws IOException { return getTargetPathHelper().getTargetPath(path, targetFs, partition, false); } protected static Path replacedPrefix(Path sourcePath, Path prefixTobeReplaced, Path prefixReplacement) { Path sourcePathWithoutSchemeAndAuthority = PathUtils.getPathWithoutSchemeAndAuthority(sourcePath); Preconditions.checkArgument(PathUtils.isAncestor(prefixTobeReplaced, sourcePathWithoutSchemeAndAuthority), "When replacing prefix, all locations must be descendants of the prefix. " + "The prefix: %s, file location: %s.", prefixTobeReplaced, sourcePathWithoutSchemeAndAuthority); Path relativePath = PathUtils.relativizePath(sourcePathWithoutSchemeAndAuthority, prefixTobeReplaced); Path result = new Path(prefixReplacement, relativePath); return result; } public FileSystem getTargetFileSystem() { return this.targetFs; } }