Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package gobblin.data.management.retention.policy; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Properties; import java.util.Set; import javax.annotation.Nullable; import org.apache.commons.lang3.reflect.ConstructorUtils; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import gobblin.data.management.retention.DatasetCleaner; import gobblin.data.management.version.DatasetVersion; /** * Implementation of {@link gobblin.data.management.retention.policy.RetentionPolicy} that allows combining different * policies through a union or intersect operation. It will combine the delete sets from each sub-policy using the * specified operation. * * <p> * For example, if there are five versions of a dataset, a, b, c, d, e, policy1 would delete versions a, b, while * policy2 would delete versions b,c, using {@link CombineRetentionPolicy} will delete versions a, b, c if the * operation is UNION, or it will delete only version b if the operation is INTERSECT. * </p> * * <p> * {@link CombineRetentionPolicy} expects the following configurations: * * gobblin.retention.combine.retention.policy.class.* : specifies the classes of the policies to combine. * can be * any value, and each such configuration defines only one class. * * gobblin.retention.combine.retention.policy.delete.sets.combine.operation : operation used to combine delete * sets. Can be UNION or INTERSECT. * Additionally, any configuration necessary for combined policies must be specified. * </p> */ public class CombineRetentionPolicy<T extends DatasetVersion> implements RetentionPolicy<T> { public static final String RETENTION_POLICIES_PREFIX = DatasetCleaner.CONFIGURATION_KEY_PREFIX + "combine.retention.policy.class."; public static final String DELETE_SETS_COMBINE_OPERATION = DatasetCleaner.CONFIGURATION_KEY_PREFIX + "combine.retention.policy.delete.sets.combine.operation"; public enum DeletableCombineOperation { INTERSECT, UNION } private final List<RetentionPolicy<T>> retentionPolicies; private final DeletableCombineOperation combineOperation; public CombineRetentionPolicy(List<RetentionPolicy<T>> retentionPolicies, DeletableCombineOperation combineOperation) { this.combineOperation = combineOperation; this.retentionPolicies = retentionPolicies; } @SuppressWarnings("unchecked") public CombineRetentionPolicy(Properties props) throws IOException { Preconditions.checkArgument(props.containsKey(DELETE_SETS_COMBINE_OPERATION), "Combine operation not specified."); ImmutableList.Builder<RetentionPolicy<T>> builder = ImmutableList.builder(); for (String property : props.stringPropertyNames()) { if (property.startsWith(RETENTION_POLICIES_PREFIX)) { try { builder.add((RetentionPolicy<T>) ConstructorUtils .invokeConstructor(Class.forName(props.getProperty(property)), props)); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) { throw new IllegalArgumentException(e); } } } this.retentionPolicies = builder.build(); if (this.retentionPolicies.size() == 0) { throw new IOException( "No retention policies specified for " + CombineRetentionPolicy.class.getCanonicalName()); } this.combineOperation = DeletableCombineOperation .valueOf(props.getProperty(DELETE_SETS_COMBINE_OPERATION).toUpperCase()); } /** * Returns the most specific common superclass for the {@link #versionClass} of each embedded policy. */ @SuppressWarnings("unchecked") @Override public Class<T> versionClass() { if (this.retentionPolicies.size() == 1) { return (Class<T>) this.retentionPolicies.get(0).versionClass(); } Class<T> klazz = (Class<T>) this.retentionPolicies.get(0).versionClass(); for (RetentionPolicy<T> policy : this.retentionPolicies) { klazz = commonSuperclass(klazz, (Class<T>) policy.versionClass()); } return klazz; } @Override public Collection<T> listDeletableVersions(final List<T> allVersions) { List<Set<T>> candidateDeletableVersions = Lists.newArrayList( Iterables.transform(this.retentionPolicies, new Function<RetentionPolicy<T>, Set<T>>() { @SuppressWarnings("deprecation") @Nullable @Override public Set<T> apply(RetentionPolicy<T> input) { return Sets.newHashSet(input.listDeletableVersions(allVersions)); } })); switch (this.combineOperation) { case INTERSECT: return intersectDatasetVersions(candidateDeletableVersions); case UNION: return unionDatasetVersions(candidateDeletableVersions); default: throw new RuntimeException("Combine operation " + this.combineOperation + " not recognized."); } } @VisibleForTesting @SuppressWarnings("unchecked") public Class<T> commonSuperclass(Class<T> classA, Class<T> classB) { if (classA.isAssignableFrom(classB)) { // a is superclass of b, so return class of a return classA; } // a is not superclass of b. Either b is superclass of a, or they are not in same branch // find closest superclass of a that is also a superclass of b Class<?> klazz = classA; while (!klazz.isAssignableFrom(classB)) { klazz = klazz.getSuperclass(); } if (DatasetVersion.class.isAssignableFrom(klazz)) { return (Class<T>) klazz; } // this should never happen, but there for safety return (Class<T>) DatasetVersion.class; } private Set<T> intersectDatasetVersions(Collection<Set<T>> sets) { if (sets.size() <= 0) { return Sets.newHashSet(); } Iterator<Set<T>> it = sets.iterator(); Set<T> outputSet = it.next(); while (it.hasNext()) { outputSet = Sets.intersection(outputSet, it.next()); } return outputSet; } private Set<T> unionDatasetVersions(Collection<Set<T>> sets) { if (sets.size() <= 0) { return Sets.newHashSet(); } Iterator<Set<T>> it = sets.iterator(); Set<T> outputSet = it.next(); while (it.hasNext()) { outputSet = Sets.union(outputSet, it.next()); } return outputSet; } }