Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package gobblin.data.management.copy.hive; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.metadata.Partition; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import gobblin.util.PathUtils; public class HiveTargetPathHelper { /** * Specifies a root path for the data in a table. All files containing table data will be placed under this directory. * <p> * Does some token replacement in the input path. For example, if the table myTable is in DB myDatabase: * /data/$DB/$TABLE -> /data/myDatabase/myTable. * /data/$TABLE -> /data/myTable * /data -> /data/myTable * </p> * * See javadoc for {@link #getTargetPath} for further explanation. */ public static final String COPY_TARGET_TABLE_ROOT = HiveDatasetFinder.HIVE_DATASET_PREFIX + ".copy.target.table.root"; /** * These two options, in pair, specify the output location of the data files on copy * {@link #COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED} specified the prefix of the path (without Scheme and Authority ) to be replaced * {@link #COPY_TARGET_TABLE_PREFIX_REPLACEMENT} specified the replacement of {@link #COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED} * <p> * for example, if the data file is $sourceFs/data/databases/DB/Table/Snapshot/part-00000.avro , * {@link #COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED} is /data/databases * {@link #COPY_TARGET_TABLE_PREFIX_REPLACEMENT} is /data/databases/_parallel * * then, the output location for that file will be * $targetFs/data/databases/_parallel/DB/Table/Snapshot/part-00000.avro * </p> */ public static final String COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED = HiveDatasetFinder.HIVE_DATASET_PREFIX + ".copy.target.table.prefixToBeReplaced"; public static final String COPY_TARGET_TABLE_PREFIX_REPLACEMENT = HiveDatasetFinder.HIVE_DATASET_PREFIX + ".copy.target.table.prefixReplacement"; /** * Specifies that, on copy, data files for this table should all be relocated to a single directory per partition. * See javadoc for {@link #getTargetPath} for further explanation. */ public static final String RELOCATE_DATA_FILES_KEY = HiveDatasetFinder.HIVE_DATASET_PREFIX + ".copy.relocate.data.files"; public static final String DEFAULT_RELOCATE_DATA_FILES = Boolean.toString(false); private final boolean relocateDataFiles; private final Optional<Path> targetTableRoot; private final Optional<Path> targetTablePrefixTobeReplaced; private final Optional<Path> targetTablePrefixReplacement; private final HiveDataset dataset; public HiveTargetPathHelper(HiveDataset dataset) { this.dataset = dataset; this.relocateDataFiles = Boolean.valueOf( this.dataset.getProperties().getProperty(RELOCATE_DATA_FILES_KEY, DEFAULT_RELOCATE_DATA_FILES)); this.targetTableRoot = this.dataset.getProperties().containsKey(COPY_TARGET_TABLE_ROOT) ? Optional.of(resolvePath(this.dataset.getProperties().getProperty(COPY_TARGET_TABLE_ROOT), this.dataset.getTable().getDbName(), this.dataset.getTable().getTableName())) : Optional.<Path>absent(); this.targetTablePrefixTobeReplaced = this.dataset.getProperties() .containsKey(COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED) ? Optional.of(new Path( this.dataset.getProperties().getProperty(COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED))) : Optional.<Path>absent(); this.targetTablePrefixReplacement = this.dataset.getProperties() .containsKey(COPY_TARGET_TABLE_PREFIX_REPLACEMENT) ? Optional.of(new Path( this.dataset.getProperties().getProperty(COPY_TARGET_TABLE_PREFIX_REPLACEMENT))) : Optional.<Path>absent(); } private static Path addPartitionToPath(Path path, Partition partition) { for (String partitionValue : partition.getValues()) { path = new Path(path, partitionValue); } return path; } /** * Takes a path with tokens {@link #databaseToken} or {@link #tableToken} and replaces these tokens with the actual * database names and table name. For example, if db is myDatabase, table is myTable, then /data/$DB/$TABLE will be * resolved to /data/myDatabase/myTable. */ protected static Path resolvePath(String pattern, String database, String table) { pattern = pattern.replace(HiveDataset.DATABASE_TOKEN, database); if (pattern.contains(HiveDataset.TABLE_TOKEN)) { pattern = pattern.replace(HiveDataset.TABLE_TOKEN, table); return new Path(pattern); } else { return new Path(pattern, table); } } /** * Compute the target {@link Path} for a file or directory copied by Hive distcp. * * <p> * The target locations of data files for this table depend on the values of the resolved table root (e.g. * the value of {@link #COPY_TARGET_TABLE_ROOT} with tokens replaced) and {@link #RELOCATE_DATA_FILES_KEY}: * * if {@link #RELOCATE_DATA_FILES_KEY} is true, then origin file /path/to/file/myFile will be written to * /resolved/table/root/<partition>/myFile * * if {@link #COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED} and {@link #COPY_TARGET_TABLE_PREFIX_REPLACEMENT} are defined, * then the specified prefix in each file will be replaced by the specified replacement. * * otherwise, if the resolved table root is defined (e.g. {@link #COPY_TARGET_TABLE_ROOT} is defined in the * properties), we define: * origin_table_root := the deepest non glob ancestor of table.getSc().getLocation() iff getLocation() points to * a single glob. (e.g. /path/to/*/files -> /path/to). If getLocation() contains none * or multiple globs, job will fail. * relative_path := path of the file relative to origin_table_root. If the path of the file is not a descendant * of origin_table_root, job will fail. * target_path := /resolved/table/root/relative/path * This mode is useful when moving a table with a complicated directory structure to a different base directory. * * otherwise the target is identical to the origin path. * </p> * * * @param sourcePath Source path to be transformed. * @param targetFs target {@link FileSystem} * @param partition partition this file belongs to. * @param isConcreteFile true if this is a path to an existing file in HDFS. */ public Path getTargetPath(Path sourcePath, FileSystem targetFs, Optional<Partition> partition, boolean isConcreteFile) { if (this.relocateDataFiles) { Preconditions.checkArgument(this.targetTableRoot.isPresent(), "Must define %s to relocate data files.", COPY_TARGET_TABLE_ROOT); Path path = this.targetTableRoot.get(); if (partition.isPresent()) { path = addPartitionToPath(path, partition.get()); } if (!isConcreteFile) { return targetFs.makeQualified(path); } return targetFs.makeQualified(new Path(path, sourcePath.getName())); } // both prefixs must be present as the same time // can not used with option {@link #COPY_TARGET_TABLE_ROOT} if (this.targetTablePrefixTobeReplaced.isPresent() || this.targetTablePrefixReplacement.isPresent()) { Preconditions.checkState(this.targetTablePrefixTobeReplaced.isPresent(), String.format("Must specify both %s option and %s option together", COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED, COPY_TARGET_TABLE_PREFIX_REPLACEMENT)); Preconditions.checkState(this.targetTablePrefixReplacement.isPresent(), String.format("Must specify both %s option and %s option together", COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED, COPY_TARGET_TABLE_PREFIX_REPLACEMENT)); Preconditions.checkState(!this.targetTableRoot.isPresent(), String.format("Can not specify the option %s with option %s ", COPY_TARGET_TABLE_ROOT, COPY_TARGET_TABLE_PREFIX_REPLACEMENT)); Path targetPathWithoutSchemeAndAuthority = HiveCopyEntityHelper.replacedPrefix(sourcePath, this.targetTablePrefixTobeReplaced.get(), this.targetTablePrefixReplacement.get()); return targetFs.makeQualified(targetPathWithoutSchemeAndAuthority); } else if (this.targetTableRoot.isPresent()) { Preconditions.checkArgument(this.dataset.getTableRootPath().isPresent(), "Cannot move paths to a new root unless table has exactly one location."); Preconditions.checkArgument(PathUtils.isAncestor(this.dataset.getTableRootPath().get(), sourcePath), "When moving paths to a new root, all locations must be descendants of the table root location. " + "Table root location: %s, file location: %s.", this.dataset.getTableRootPath(), sourcePath); Path relativePath = PathUtils.relativizePath(sourcePath, this.dataset.getTableRootPath().get()); return targetFs.makeQualified(new Path(this.targetTableRoot.get(), relativePath)); } else { return targetFs.makeQualified(PathUtils.getPathWithoutSchemeAndAuthority(sourcePath)); } } }