Java tutorial
/* * Copyright 2015 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package co.cask.cdap.internal.app.runtime.artifact; import co.cask.cdap.api.artifact.ApplicationClass; import co.cask.cdap.api.artifact.ArtifactClasses; import co.cask.cdap.api.artifact.ArtifactVersion; import co.cask.cdap.api.common.Bytes; import co.cask.cdap.api.data.schema.Schema; import co.cask.cdap.api.dataset.DatasetDefinition; import co.cask.cdap.api.dataset.DatasetManagementException; import co.cask.cdap.api.dataset.DatasetProperties; import co.cask.cdap.api.dataset.table.ConflictDetection; import co.cask.cdap.api.dataset.table.Row; import co.cask.cdap.api.dataset.table.Scan; import co.cask.cdap.api.dataset.table.Scanner; import co.cask.cdap.api.dataset.table.Table; import co.cask.cdap.api.plugin.PluginClass; import co.cask.cdap.common.ArtifactAlreadyExistsException; import co.cask.cdap.common.ArtifactNotFoundException; import co.cask.cdap.common.io.Locations; import co.cask.cdap.common.namespace.NamespacedLocationFactory; import co.cask.cdap.common.utils.ImmutablePair; import co.cask.cdap.data2.datafabric.dataset.DatasetsUtil; import co.cask.cdap.data2.dataset2.DatasetFramework; import co.cask.cdap.data2.dataset2.tx.DatasetContext; import co.cask.cdap.data2.dataset2.tx.Transactional; import co.cask.cdap.internal.app.runtime.plugin.PluginNotExistsException; import co.cask.cdap.internal.io.SchemaTypeAdapter; import co.cask.cdap.proto.Id; import co.cask.cdap.proto.artifact.ArtifactRange; import co.cask.cdap.proto.artifact.InvalidArtifactRangeException; import co.cask.cdap.proto.id.Ids; import co.cask.cdap.proto.id.NamespaceId; import co.cask.tephra.TransactionConflictException; import co.cask.tephra.TransactionExecutor; import co.cask.tephra.TransactionExecutorFactory; import co.cask.tephra.TransactionFailureException; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.base.Splitter; import com.google.common.base.Supplier; import com.google.common.base.Throwables; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.io.ByteStreams; import com.google.common.io.InputSupplier; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonDeserializationContext; import com.google.gson.JsonDeserializer; import com.google.gson.JsonElement; import com.google.gson.JsonParseException; import com.google.gson.JsonPrimitive; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; import com.google.inject.Inject; import org.apache.twill.filesystem.Location; import org.apache.twill.filesystem.LocationFactory; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.lang.reflect.Type; import java.net.URI; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; /** * This class manages artifacts as well as metadata for each artifact. Artifacts and their metadata cannot be changed * once they are written, with the exception of snapshot versions. An Artifact can contain * plugin classes and/or application classes. We may want to extend this to include other types of classes, such * as datasets. * * Every time an artifact is added, the artifact contents are stored at a base location based on its id: * /namespaces/{namespace-id}/artifacts/{artifact-name}/{artifact-version} * Several metadata writes are then performed. * * The first adds metadata about the artifact, with * rowkey r:{namespace}:{artifact-name}, column {artifact-version}, and ArtifactData as the value * * The second adds metadata about any Application Class contained in the artifact, with * rowkey a:{namespace}:{classname}, column {artifact-name}:{artifact-version}, and AppClass as the value * * The third adds metadata about any Plugin contained in the artifact, with * rowkey p:{parent-namespace}:{parent-name}:{plugin-type}:{plugin-name}, * column {artifact-namespace}:{artifact-name}:{artifact-version}, * and PluginData as the value * * For example, suppose we add a system artifact etlbatch-3.1.0, which contains an ETLBatch application class. * The meta table will look like: * * rowkey columns * a:system:ETLBatch etlbatch:3.1.0 -> {AppData} * r:system:etlbatch 3.1.0 -> {ArtifactData} * * After that, a system artifact etlbatch-lib-3.1.0 is added, which extends etlbatch and contains * stream sink and table sink plugins. The meta table will look like: * * rowkey columns * a:system:ETLBatch etlbatch:3.1.0 -> {AppData} * p:system:etlbatch:sink:stream system:etlbatch-lib:3.1.0 -> {PluginData} * p:system:etlbatch:sink:table system:etlbatch-lib:3.1.0 -> {PluginData} * r:system:etlbatch 3.1.0 -> {ArtifactData} * r:system:etlbatch-lib 3.1.0 -> {ArtifactData} * * Finally a user adds artifact custom-sources-1.0.0 to the default namespace, * which extends etlbatch and contains a db source plugin. The meta table will look like: * * rowkey columns * a:system:ETLBatch etlbatch:3.1.0 -> {AppData} * p:system:etlbatch:sink:stream system:etlbatch-lib:3.1.0 -> {PluginData} * p:system:etlbatch:sink:table system:etlbatch-lib:3.1.0 -> {PluginData} * p:system:etlbatch:source:db default:custom-sources:1.0.0 -> {PluginData} * r:default:custom-sources 1.0.0 -> {ArtifactData} * r:system:etlbatch 3.1.0 -> {ArtifactData} * r:system:etlbatch-lib 3.1.0 -> {ArtifactData} * * With this schema we can perform a scan to look up AppClasses, a scan to look up plugins that extend a specific * artifact, and a scan to look up artifacts. */ public class ArtifactStore { private static final String ARTIFACTS_PATH = "artifacts"; private static final String ARTIFACT_PREFIX = "r"; private static final String PLUGIN_PREFIX = "p"; private static final String APPCLASS_PREFIX = "a"; private static final Id.DatasetInstance META_ID = Id.DatasetInstance.from(Id.Namespace.SYSTEM, "artifact.meta"); private static final DatasetProperties META_PROPERTIES = DatasetProperties.builder() .add(Table.PROPERTY_CONFLICT_LEVEL, ConflictDetection.COLUMN.name()).build(); private final LocationFactory locationFactory; private final NamespacedLocationFactory namespacedLocationFactory; private final Transactional<DatasetContext<Table>, Table> metaTable; private final Gson gson; @Inject ArtifactStore(final DatasetFramework datasetFramework, NamespacedLocationFactory namespacedLocationFactory, LocationFactory locationFactory, TransactionExecutorFactory txExecutorFactory) { this.locationFactory = locationFactory; this.namespacedLocationFactory = namespacedLocationFactory; this.gson = new GsonBuilder().registerTypeAdapter(Schema.class, new SchemaTypeAdapter()) .registerTypeAdapter(ArtifactRange.class, new ArtifactRangeCodec()).create(); this.metaTable = Transactional.of(txExecutorFactory, new Supplier<DatasetContext<Table>>() { @Override public DatasetContext<Table> get() { try { return DatasetContext.of((Table) DatasetsUtil.getOrCreateDataset(datasetFramework, META_ID, Table.class.getName(), META_PROPERTIES, DatasetDefinition.NO_ARGUMENTS, null)); } catch (Exception e) { // there's nothing much we can do here throw Throwables.propagate(e); } } }); } /** * Adds datasets and types to the given {@link DatasetFramework} used by artifact store. * * @param framework framework to add types and datasets to */ public static void setupDatasets(DatasetFramework framework) throws IOException, DatasetManagementException { framework.addInstance(Table.class.getName(), META_ID, META_PROPERTIES); } /** * Get information about all artifacts in the given namespace. If there are no artifacts in the namespace, * this will return an empty list. Note that existence of the namespace is not checked. * * @param namespace the namespace to get artifact information about * @return unmodifiable list of artifact info about every artifact in the given namespace * @throws IOException if there was an exception reading the artifact information from the metastore */ public List<ArtifactDetail> getArtifacts(final NamespaceId namespace) throws IOException { return metaTable .executeUnchecked(new TransactionExecutor.Function<DatasetContext<Table>, List<ArtifactDetail>>() { @Override public List<ArtifactDetail> apply(DatasetContext<Table> context) throws Exception { List<ArtifactDetail> artifacts = Lists.newArrayList(); Scanner scanner = context.get().scan(scanArtifacts(namespace)); Row row; while ((row = scanner.next()) != null) { addArtifactsToList(artifacts, row); } scanner.close(); return Collections.unmodifiableList(artifacts); } }); } /** * Get all artifacts that match artifacts in the given ranges. * * @param range the range to match artifacts in * @return an unmodifiable list of all artifacts that match the given ranges. If none exist, an empty list * is returned */ public List<ArtifactDetail> getArtifacts(final ArtifactRange range) { return metaTable .executeUnchecked(new TransactionExecutor.Function<DatasetContext<Table>, List<ArtifactDetail>>() { @Override public List<ArtifactDetail> apply(DatasetContext<Table> context) throws Exception { List<ArtifactDetail> artifacts = Lists.newArrayList(); Table table = context.get(); ArtifactKey artifactKey = new ArtifactKey(range.getNamespace().toEntityId(), range.getName()); Row row = table.get(artifactKey.getRowKey()); for (Map.Entry<byte[], byte[]> columnEntry : row.getColumns().entrySet()) { String version = Bytes.toString(columnEntry.getKey()); if (range.versionIsInRange(new ArtifactVersion(version))) { ArtifactData data = gson.fromJson(Bytes.toString(columnEntry.getValue()), ArtifactData.class); Id.Artifact artifactId = Id.Artifact.from(artifactKey.namespace.toId(), artifactKey.name, version); artifacts.add(new ArtifactDetail(new ArtifactDescriptor(artifactId.toArtifactId(), locationFactory.create(data.locationURI)), data.meta)); } } return Collections.unmodifiableList(artifacts); } }); } /** * Get information about all versions of the given artifact. * * @param namespace the namespace to get artifacts from * @param artifactName the name of the artifact to get * @return unmodifiable list of information about all versions of the given artifact * @throws ArtifactNotFoundException if no version of the given artifact exists * @throws IOException if there was an exception reading the artifact information from the metastore */ public List<ArtifactDetail> getArtifacts(final NamespaceId namespace, final String artifactName) throws ArtifactNotFoundException, IOException { List<ArtifactDetail> artifacts = metaTable .executeUnchecked(new TransactionExecutor.Function<DatasetContext<Table>, List<ArtifactDetail>>() { @Override public List<ArtifactDetail> apply(DatasetContext<Table> context) throws Exception { List<ArtifactDetail> archives = Lists.newArrayList(); ArtifactKey artifactKey = new ArtifactKey(namespace, artifactName); Row row = context.get().get(artifactKey.getRowKey()); if (!row.isEmpty()) { addArtifactsToList(archives, row); } return archives; } }); if (artifacts.isEmpty()) { throw new ArtifactNotFoundException(namespace.toId(), artifactName); } return Collections.unmodifiableList(artifacts); } /** * Get information about the given artifact. * * @param artifactId the artifact to get * @return information about the artifact * @throws ArtifactNotFoundException if the given artifact does not exist * @throws IOException if there was an exception reading the artifact information from the metastore */ public ArtifactDetail getArtifact(final Id.Artifact artifactId) throws ArtifactNotFoundException, IOException { ArtifactData data = metaTable .executeUnchecked(new TransactionExecutor.Function<DatasetContext<Table>, ArtifactData>() { @Override public ArtifactData apply(DatasetContext<Table> context) throws Exception { ArtifactCell artifactCell = new ArtifactCell(artifactId); byte[] value = context.get().get(artifactCell.rowkey, artifactCell.column); return value == null ? null : gson.fromJson(Bytes.toString(value), ArtifactData.class); } }); if (data == null) { throw new ArtifactNotFoundException(artifactId); } return new ArtifactDetail( new ArtifactDescriptor(artifactId.toArtifactId(), locationFactory.create(data.locationURI)), data.meta); } /** * Get all application classes that belong to the specified namespace. * Results are returned as a sorted map from artifact to application classes in that artifact. * Map entries are sorted by the artifact. * * @param namespace the namespace from which to get application classes * @return an unmodifiable map of artifact to a list of all application classes in that artifact. * The map will never be null. If there are no application classes, an empty map will be returned. */ public SortedMap<ArtifactDescriptor, List<ApplicationClass>> getApplicationClasses( final NamespaceId namespace) { return metaTable.executeUnchecked( new TransactionExecutor.Function<DatasetContext<Table>, SortedMap<ArtifactDescriptor, List<ApplicationClass>>>() { @Override public SortedMap<ArtifactDescriptor, List<ApplicationClass>> apply( DatasetContext<Table> context) { SortedMap<ArtifactDescriptor, List<ApplicationClass>> result = Maps.newTreeMap(); Scanner scanner = context.get().scan(scanAppClasses(namespace)); Row row; while ((row = scanner.next()) != null) { // columns are {artifact-name}:{artifact-version}. vals are serialized AppData for (Map.Entry<byte[], byte[]> column : row.getColumns().entrySet()) { ArtifactColumn artifactColumn = ArtifactColumn.parse(column.getKey()); AppData appData = gson.fromJson(Bytes.toString(column.getValue()), AppData.class); ArtifactDescriptor artifactDescriptor = new ArtifactDescriptor( artifactColumn.artifactId.toArtifactId(), locationFactory.create(appData.artifactLocationURI)); List<ApplicationClass> existingAppClasses = result.get(artifactDescriptor); if (existingAppClasses == null) { existingAppClasses = new ArrayList<>(); result.put(artifactDescriptor, existingAppClasses); } existingAppClasses.add(appData.appClass); } } scanner.close(); return Collections.unmodifiableSortedMap(result); } }); } /** * Get all application classes that belong to the specified namespace of the specified classname. * Results are returned as a sorted map from artifact to application classes in that artifact. * Map entries are sorted by the artifact. * * @param namespace the namespace from which to get application classes * @param className the classname of application classes to get * @return an unmodifiable map of artifact the application classes in that artifact. * The map will never be null. If there are no application classes, an empty map will be returned. */ public SortedMap<ArtifactDescriptor, ApplicationClass> getApplicationClasses(final NamespaceId namespace, final String className) { return metaTable.executeUnchecked( new TransactionExecutor.Function<DatasetContext<Table>, SortedMap<ArtifactDescriptor, ApplicationClass>>() { @Override public SortedMap<ArtifactDescriptor, ApplicationClass> apply(DatasetContext<Table> context) { SortedMap<ArtifactDescriptor, ApplicationClass> result = Maps.newTreeMap(); Row row = context.get().get(new AppClassKey(namespace, className).getRowKey()); if (!row.isEmpty()) { // columns are {artifact-name}:{artifact-version}. vals are serialized AppData for (Map.Entry<byte[], byte[]> column : row.getColumns().entrySet()) { ArtifactColumn artifactColumn = ArtifactColumn.parse(column.getKey()); AppData appData = gson.fromJson(Bytes.toString(column.getValue()), AppData.class); ArtifactDescriptor artifactDescriptor = new ArtifactDescriptor( artifactColumn.artifactId.toArtifactId(), locationFactory.create(appData.artifactLocationURI)); result.put(artifactDescriptor, appData.appClass); } } return Collections.unmodifiableSortedMap(result); } }); } /** * Get all plugin classes that extend the given parent artifact. * Results are returned as a sorted map from plugin artifact to plugins in that artifact. * Map entries are sorted by the artifact id of the plugin. * * @param parentArtifactId the id of the artifact to find plugins for * @return an unmodifiable map of plugin artifact to plugin classes for all plugin classes accessible by the given * artifact. The map will never be null. If there are no plugin classes, an empty map will be returned. * @throws ArtifactNotFoundException if the artifact to find plugins for does not exist * @throws IOException if there was an exception reading metadata from the metastore */ public SortedMap<ArtifactDescriptor, Set<PluginClass>> getPluginClasses(final NamespaceId namespace, final Id.Artifact parentArtifactId) throws ArtifactNotFoundException, IOException { SortedMap<ArtifactDescriptor, Set<PluginClass>> pluginClasses = metaTable.executeUnchecked( new TransactionExecutor.Function<DatasetContext<Table>, SortedMap<ArtifactDescriptor, Set<PluginClass>>>() { @Override public SortedMap<ArtifactDescriptor, Set<PluginClass>> apply(DatasetContext<Table> context) throws Exception { Table table = context.get(); SortedMap<ArtifactDescriptor, Set<PluginClass>> result = getPluginsInArtifact(table, parentArtifactId); if (result == null) { return null; } // should be able to scan by column prefix as well... instead, we have to filter out by namespace Scanner scanner = table.scan(scanPlugins(parentArtifactId)); Row row; while ((row = scanner.next()) != null) { addPluginsToMap(namespace, parentArtifactId, result, row); } scanner.close(); return Collections.unmodifiableSortedMap(result); } }); if (pluginClasses == null) { throw new ArtifactNotFoundException(parentArtifactId); } return pluginClasses; } /** * Get all plugin classes of the given type that extend the given parent artifact. * Results are returned as a map from plugin artifact to plugins in that artifact. * * @param parentArtifactId the id of the artifact to find plugins for * @param type the type of plugin to look for * @return an unmodifiable map of plugin artifact to plugin classes for all plugin classes accessible by the * given artifact. The map will never be null. If there are no plugin classes, an empty map will be returned. * @throws ArtifactNotFoundException if the artifact to find plugins for does not exist * @throws IOException if there was an exception reading metadata from the metastore */ public SortedMap<ArtifactDescriptor, Set<PluginClass>> getPluginClasses(final NamespaceId namespace, final Id.Artifact parentArtifactId, final String type) throws IOException, ArtifactNotFoundException { SortedMap<ArtifactDescriptor, Set<PluginClass>> pluginClasses = metaTable.executeUnchecked( new TransactionExecutor.Function<DatasetContext<Table>, SortedMap<ArtifactDescriptor, Set<PluginClass>>>() { @Override public SortedMap<ArtifactDescriptor, Set<PluginClass>> apply(DatasetContext<Table> context) throws Exception { Table table = context.get(); SortedMap<ArtifactDescriptor, Set<PluginClass>> result = getPluginsInArtifact(table, parentArtifactId, new Predicate<PluginClass>() { @Override public boolean apply(PluginClass input) { return type.equals(input.getType()); } }); if (result == null) { return null; } Scanner scanner = table.scan(scanPlugins(parentArtifactId, type)); Row row; while ((row = scanner.next()) != null) { addPluginsToMap(namespace, parentArtifactId, result, row); } scanner.close(); return Collections.unmodifiableSortedMap(result); } }); if (pluginClasses == null) { throw new ArtifactNotFoundException(parentArtifactId); } return pluginClasses; } /** * Get all plugin classes of the given type and name that extend the given parent artifact. * Results are returned as a map from plugin artifact to plugins in that artifact. * * @param parentArtifactId the id of the artifact to find plugins for * @param type the type of plugin to look for * @param name the name of the plugin to look for * @return an unmodifiable map of plugin artifact to plugin classes of the given type and name, accessible by the * given artifact. The map will never be null, and will never be empty. * @throws PluginNotExistsException if no plugin with the given type and name exists in the namespace * @throws IOException if there was an exception reading metadata from the metastore */ public SortedMap<ArtifactDescriptor, PluginClass> getPluginClasses(final NamespaceId namespace, final Id.Artifact parentArtifactId, final String type, final String name) throws IOException, ArtifactNotFoundException, PluginNotExistsException { SortedMap<ArtifactDescriptor, PluginClass> plugins = metaTable.executeUnchecked( new TransactionExecutor.Function<DatasetContext<Table>, SortedMap<ArtifactDescriptor, PluginClass>>() { @Override public SortedMap<ArtifactDescriptor, PluginClass> apply(DatasetContext<Table> context) throws Exception { Table table = context.get(); SortedMap<ArtifactDescriptor, PluginClass> result = new TreeMap<>(); // check parent exists ArtifactCell parentCell = new ArtifactCell(parentArtifactId); byte[] parentDataBytes = table.get(parentCell.rowkey, parentCell.column); if (parentDataBytes == null) { return null; } // check if any plugins of that type and name exist in the parent artifact already ArtifactData parentData = gson.fromJson(Bytes.toString(parentDataBytes), ArtifactData.class); Set<PluginClass> parentPlugins = parentData.meta.getClasses().getPlugins(); for (PluginClass pluginClass : parentPlugins) { if (pluginClass.getName().equals(name) && pluginClass.getType().equals(type)) { ArtifactDescriptor parentDescriptor = new ArtifactDescriptor( parentArtifactId.toArtifactId(), locationFactory.create(parentData.locationURI)); result.put(parentDescriptor, pluginClass); break; } } PluginKey pluginKey = new PluginKey(parentArtifactId.getNamespace(), parentArtifactId.getName(), type, name); Row row = context.get().get(pluginKey.getRowKey()); if (!row.isEmpty()) { // column is the artifact name and version, value is the serialized PluginClass for (Map.Entry<byte[], byte[]> column : row.getColumns().entrySet()) { ImmutablePair<ArtifactDescriptor, PluginClass> pluginEntry = getPluginEntry( namespace, parentArtifactId, column); if (pluginEntry != null) { result.put(pluginEntry.getFirst(), pluginEntry.getSecond()); } } } return result; } }); if (plugins == null) { throw new ArtifactNotFoundException(parentArtifactId); } if (plugins.isEmpty()) { throw new PluginNotExistsException(parentArtifactId.getNamespace(), type, name); } return Collections.unmodifiableSortedMap(plugins); } /** * Update artifact properties using an update function. Functions will receive an immutable map. * * @param artifactId the id of the artifact to add * @param updateFunction the function used to update existing properties * @throws ArtifactNotFoundException if the artifact does not exist * @throws IOException if there was an exception writing the properties to the metastore */ public void updateArtifactProperties(final Id.Artifact artifactId, final Function<Map<String, String>, Map<String, String>> updateFunction) throws ArtifactNotFoundException, IOException { try { boolean exists = metaTable.execute(new TransactionExecutor.Function<DatasetContext<Table>, Boolean>() { @Override public Boolean apply(DatasetContext<Table> context) throws Exception { Table table = context.get(); ArtifactCell artifactCell = new ArtifactCell(artifactId); byte[] existingMetaBytes = table.get(artifactCell.rowkey, artifactCell.column); if (existingMetaBytes == null) { return false; } ArtifactData old = gson.fromJson(Bytes.toString(existingMetaBytes), ArtifactData.class); ArtifactMeta updatedMeta = new ArtifactMeta(old.meta.getClasses(), old.meta.getUsableBy(), updateFunction.apply(old.meta.getProperties())); ArtifactData updatedData = new ArtifactData(locationFactory.create(old.locationURI), updatedMeta); // write artifact metadata table.put(artifactCell.rowkey, artifactCell.column, Bytes.toBytes(gson.toJson(updatedData))); return true; } }); if (!exists) { throw new ArtifactNotFoundException(artifactId); } } catch (TransactionFailureException | InterruptedException e) { throw new IOException(e); } } /** * Write the artifact and its metadata to the store. Once added, artifacts cannot be changed unless they are * snapshot versions. * * @param artifactId the id of the artifact to add * @param artifactMeta the metadata for the artifact * @param artifactContentSupplier the supplier for the input stream of the contents of the artifact * @return detail about the newly added artifact * @throws WriteConflictException if the artifact is already currently being written * @throws ArtifactAlreadyExistsException if a non-snapshot version of the artifact already exists * @throws IOException if there was an exception persisting the artifact contents to the filesystem, * of persisting the artifact metadata to the metastore */ public ArtifactDetail write(final Id.Artifact artifactId, final ArtifactMeta artifactMeta, final InputSupplier<? extends InputStream> artifactContentSupplier) throws WriteConflictException, ArtifactAlreadyExistsException, IOException { // if we're not a snapshot version, check that the artifact doesn't exist already. final ArtifactCell artifactCell = new ArtifactCell(artifactId); if (!artifactId.getVersion().isSnapshot()) { byte[] existingMeta = metaTable .executeUnchecked(new TransactionExecutor.Function<DatasetContext<Table>, byte[]>() { @Override public byte[] apply(DatasetContext<Table> context) throws Exception { Table table = context.get(); return table.get(artifactCell.rowkey, artifactCell.column); } }); if (existingMeta != null) { throw new ArtifactAlreadyExistsException(artifactId); } } Location fileDirectory = namespacedLocationFactory.get(artifactId.getNamespace(), ARTIFACTS_PATH) .append(artifactId.getName()); Locations.mkdirsIfNotExists(fileDirectory); // write the file contents final Location destination = fileDirectory.append(artifactId.getVersion().getVersion()).getTempFile(".jar"); try (InputStream artifactContents = artifactContentSupplier.getInput(); OutputStream destinationStream = destination.getOutputStream()) { ByteStreams.copy(artifactContents, destinationStream); } // now try and write the metadata for the artifact try { boolean written = metaTable.execute(new TransactionExecutor.Function<DatasetContext<Table>, Boolean>() { @Override public Boolean apply(DatasetContext<Table> context) throws Exception { Table table = context.get(); // we have to check that the metadata doesn't exist again since somebody else may have written // the artifact while we were copying the artifact to the filesystem. byte[] existingMetaBytes = table.get(artifactCell.rowkey, artifactCell.column); boolean isSnapshot = artifactId.getVersion().isSnapshot(); if (existingMetaBytes != null && !isSnapshot) { // non-snapshot artifacts are immutable. If there is existing metadata, stop here. return false; } ArtifactData data = new ArtifactData(destination, artifactMeta); // cleanup existing metadata if it exists and this is a snapshot // if we are overwriting a previous snapshot, need to clean up the old snapshot data // this means cleaning up the old jar, and deleting plugin and app rows. if (existingMetaBytes != null) { deleteMeta(table, artifactId, existingMetaBytes); } // write artifact metadata writeMeta(table, artifactId, data); return true; } }); if (!written) { throw new ArtifactAlreadyExistsException(artifactId); } } catch (TransactionConflictException e) { destination.delete(); throw new WriteConflictException(artifactId); } catch (TransactionFailureException | InterruptedException e) { destination.delete(); throw new IOException(e); } return new ArtifactDetail(new ArtifactDescriptor(artifactId.toArtifactId(), destination), artifactMeta); } /** * Delete the specified artifact. Programs that use the artifact will no longer be able to start. * * @param artifactId the id of the artifact to delete * @throws IOException if there was an IO error deleting the metadata or the actual artifact */ public void delete(final Id.Artifact artifactId) throws IOException { // delete everything in a transaction metaTable.executeUnchecked(new TransactionExecutor.Function<DatasetContext<Table>, Void>() { @Override public Void apply(DatasetContext<Table> context) throws Exception { Table table = context.get(); // first look up details to get plugins and apps in the artifact ArtifactCell artifactCell = new ArtifactCell(artifactId); byte[] detailBytes = table.get(artifactCell.rowkey, artifactCell.column); if (detailBytes == null) { // ok there is nothing to delete, we're done return null; } deleteMeta(table, artifactId, detailBytes); return null; } }); } /** * Clear all data in the given namespace. Used only in unit tests. * * @param namespace the namespace to delete data in * @throws IOException if there was some problem deleting the data */ @VisibleForTesting void clear(final NamespaceId namespace) throws IOException { namespacedLocationFactory.get(namespace.toId(), ARTIFACTS_PATH).delete(true); metaTable.executeUnchecked(new TransactionExecutor.Function<DatasetContext<Table>, Void>() { @Override public Void apply(DatasetContext<Table> context) throws Exception { Table table = context.get(); // delete all rows about artifacts in the namespace Scanner scanner = table.scan(scanArtifacts(namespace)); Row row; while ((row = scanner.next()) != null) { table.delete(row.getRow()); } scanner.close(); // delete all rows about artifacts in the namespace and the plugins they have access to Scan pluginsScan = new Scan( Bytes.toBytes(String.format("%s:%s:", PLUGIN_PREFIX, namespace.getNamespace())), Bytes.toBytes(String.format("%s:%s;", PLUGIN_PREFIX, namespace.getNamespace()))); scanner = table.scan(pluginsScan); while ((row = scanner.next()) != null) { table.delete(row.getRow()); } scanner.close(); // delete app classes in this namespace scanner = table.scan(scanAppClasses(namespace)); while ((row = scanner.next()) != null) { table.delete(row.getRow()); } scanner.close(); // delete plugins in this namespace from system artifacts // for example, if there was an artifact in this namespace that extends a system artifact Scan systemPluginsScan = new Scan( Bytes.toBytes(String.format("%s:%s:", PLUGIN_PREFIX, Id.Namespace.SYSTEM.getId())), Bytes.toBytes(String.format("%s:%s;", PLUGIN_PREFIX, Id.Namespace.SYSTEM.getId()))); scanner = table.scan(systemPluginsScan); while ((row = scanner.next()) != null) { for (Map.Entry<byte[], byte[]> columnVal : row.getColumns().entrySet()) { // the column is the id of the artifact the plugin is from ArtifactColumn column = ArtifactColumn.parse(columnVal.getKey()); // if the plugin artifact is in the namespace we're deleting, delete this column. if (column.artifactId.getNamespace().equals(namespace)) { table.delete(row.getRow(), column.getColumn()); } } } scanner.close(); return null; } }); } // write a new artifact snapshot and clean up the old snapshot data private void writeMeta(Table table, Id.Artifact artifactId, ArtifactData data) throws IOException { ArtifactCell artifactCell = new ArtifactCell(artifactId); table.put(artifactCell.rowkey, artifactCell.column, Bytes.toBytes(gson.toJson(data))); // column for plugin meta and app meta. {artifact-name}:{artifact-version} // does not need to contain namespace because namespace is in the rowkey byte[] artifactColumn = new ArtifactColumn(artifactId).getColumn(); ArtifactClasses classes = data.meta.getClasses(); Location artifactLocation = locationFactory.create(data.locationURI); // write pluginClass metadata for (PluginClass pluginClass : classes.getPlugins()) { // write metadata for each artifact this plugin extends for (ArtifactRange artifactRange : data.meta.getUsableBy()) { // p:{namespace}:{type}:{name} PluginKey pluginKey = new PluginKey(artifactRange.getNamespace(), artifactRange.getName(), pluginClass.getType(), pluginClass.getName()); byte[] pluginDataBytes = Bytes .toBytes(gson.toJson(new PluginData(pluginClass, artifactRange, artifactLocation))); table.put(pluginKey.getRowKey(), artifactColumn, pluginDataBytes); } } // write appClass metadata for (ApplicationClass appClass : classes.getApps()) { // a:{namespace}:{classname} AppClassKey appClassKey = new AppClassKey(artifactId.getNamespace().toEntityId(), appClass.getClassName()); byte[] appDataBytes = Bytes.toBytes(gson.toJson(new AppData(appClass, artifactLocation))); table.put(appClassKey.getRowKey(), artifactColumn, appDataBytes); } } private void deleteMeta(Table table, Id.Artifact artifactId, byte[] oldData) throws IOException { // delete old artifact data ArtifactCell artifactCell = new ArtifactCell(artifactId); table.delete(artifactCell.rowkey, artifactCell.column); // delete old plugins ArtifactData oldMeta = gson.fromJson(Bytes.toString(oldData), ArtifactData.class); byte[] artifactColumn = new ArtifactColumn(artifactId).getColumn(); for (PluginClass pluginClass : oldMeta.meta.getClasses().getPlugins()) { // delete metadata for each artifact this plugin extends for (ArtifactRange artifactRange : oldMeta.meta.getUsableBy()) { // p:{namespace}:{type}:{name} PluginKey pluginKey = new PluginKey(artifactRange.getNamespace(), artifactRange.getName(), pluginClass.getType(), pluginClass.getName()); table.delete(pluginKey.getRowKey(), artifactColumn); } } // delete old appclass metadata for (ApplicationClass appClass : oldMeta.meta.getClasses().getApps()) { AppClassKey appClassKey = new AppClassKey(artifactId.getNamespace().toEntityId(), appClass.getClassName()); table.delete(appClassKey.getRowKey(), artifactColumn); } // delete the old jar file locationFactory.create(oldMeta.locationURI).delete(); } private SortedMap<ArtifactDescriptor, Set<PluginClass>> getPluginsInArtifact(Table table, Id.Artifact artifactId) { return getPluginsInArtifact(table, artifactId, Predicates.<PluginClass>alwaysTrue()); } private SortedMap<ArtifactDescriptor, Set<PluginClass>> getPluginsInArtifact(Table table, Id.Artifact artifactId, Predicate<PluginClass> filter) { SortedMap<ArtifactDescriptor, Set<PluginClass>> result = new TreeMap<>(); // Make sure the artifact exists ArtifactCell parentCell = new ArtifactCell(artifactId); byte[] parentDataBytes = table.get(parentCell.rowkey, parentCell.column); if (parentDataBytes == null) { return null; } // include any plugin classes that are inside the artifact itself ArtifactData parentData = gson.fromJson(Bytes.toString(parentDataBytes), ArtifactData.class); Set<PluginClass> parentPlugins = parentData.meta.getClasses().getPlugins(); Set<PluginClass> filteredPlugins = Sets.newLinkedHashSet(Iterables.filter(parentPlugins, filter)); if (!filteredPlugins.isEmpty()) { Location parentLocation = locationFactory.create(parentData.locationURI); ArtifactDescriptor descriptor = new ArtifactDescriptor(artifactId.toArtifactId(), parentLocation); result.put(descriptor, filteredPlugins); } return result; } private void addArtifactsToList(List<ArtifactDetail> artifactDetails, Row row) throws IOException { ArtifactKey artifactKey = ArtifactKey.parse(row.getRow()); for (Map.Entry<byte[], byte[]> columnVal : row.getColumns().entrySet()) { String version = Bytes.toString(columnVal.getKey()); ArtifactData data = gson.fromJson(Bytes.toString(columnVal.getValue()), ArtifactData.class); Id.Artifact artifactId = Id.Artifact.from(artifactKey.namespace.toId(), artifactKey.name, version); artifactDetails.add(new ArtifactDetail( new ArtifactDescriptor(artifactId.toArtifactId(), locationFactory.create(data.locationURI)), data.meta)); } } // this method examines all plugins in the given row and checks if they extend the given parent artifact // and are from an artifact in the given namespace. // if so, information about the plugin artifact and the plugin details are added to the given map. private void addPluginsToMap(NamespaceId namespace, Id.Artifact parentArtifactId, SortedMap<ArtifactDescriptor, Set<PluginClass>> map, Row row) throws IOException { // column is the artifact namespace, name, and version. value is the serialized PluginData for (Map.Entry<byte[], byte[]> column : row.getColumns().entrySet()) { ImmutablePair<ArtifactDescriptor, PluginClass> pluginEntry = getPluginEntry(namespace, parentArtifactId, column); if (pluginEntry != null) { ArtifactDescriptor artifactDescriptor = pluginEntry.getFirst(); if (!map.containsKey(artifactDescriptor)) { map.put(artifactDescriptor, Sets.<PluginClass>newHashSet()); } map.get(artifactDescriptor).add(pluginEntry.getSecond()); } } } /** * Decode the PluginClass from the table column if it is from an artifact in the given namespace and * extends the given parent artifact. If the plugin's artifact is not in the given namespace, or it does not * extend the given parent artifact, return null. */ private ImmutablePair<ArtifactDescriptor, PluginClass> getPluginEntry(NamespaceId namespace, Id.Artifact parentArtifactId, Map.Entry<byte[], byte[]> column) { // column is the artifact namespace, name, and version. value is the serialized PluginData ArtifactColumn artifactColumn = ArtifactColumn.parse(column.getKey()); Id.Namespace artifactNamespace = artifactColumn.artifactId.getNamespace(); // filter out plugins whose artifacts are not in the system namespace and not in this namespace if (!Id.Namespace.SYSTEM.equals(artifactNamespace) && !artifactNamespace.equals(namespace.toId())) { return null; } PluginData pluginData = gson.fromJson(Bytes.toString(column.getValue()), PluginData.class); // filter out plugins that don't extend this version of the parent artifact if (pluginData.usableBy.versionIsInRange(parentArtifactId.getVersion())) { ArtifactDescriptor artifactDescriptor = new ArtifactDescriptor(artifactColumn.artifactId.toArtifactId(), locationFactory.create(pluginData.artifactLocationURI)); return ImmutablePair.of(artifactDescriptor, pluginData.pluginClass); } return null; } private Scan scanArtifacts(NamespaceId namespace) { return new Scan(Bytes.toBytes(String.format("%s:%s:", ARTIFACT_PREFIX, namespace.getNamespace())), Bytes.toBytes(String.format("%s:%s;", ARTIFACT_PREFIX, namespace.getNamespace()))); } private Scan scanPlugins(Id.Artifact parentArtifactId) { return new Scan( Bytes.toBytes(String.format("%s:%s:%s:", PLUGIN_PREFIX, parentArtifactId.getNamespace().getId(), parentArtifactId.getName())), Bytes.toBytes(String.format("%s:%s:%s;", PLUGIN_PREFIX, parentArtifactId.getNamespace().getId(), parentArtifactId.getName()))); } private Scan scanPlugins(Id.Artifact parentArtifactId, String type) { return new Scan( Bytes.toBytes(String.format("%s:%s:%s:%s:", PLUGIN_PREFIX, parentArtifactId.getNamespace().getId(), parentArtifactId.getName(), type)), Bytes.toBytes(String.format("%s:%s:%s:%s;", PLUGIN_PREFIX, parentArtifactId.getNamespace().getId(), parentArtifactId.getName(), type))); } private Scan scanAppClasses(NamespaceId namespace) { return new Scan(Bytes.toBytes(String.format("%s:%s:", APPCLASS_PREFIX, namespace.getNamespace())), Bytes.toBytes(String.format("%s:%s;", APPCLASS_PREFIX, namespace.getNamespace()))); } private static class AppClassKey { private final NamespaceId namespace; private final String className; public AppClassKey(NamespaceId namespace, String className) { this.namespace = namespace; this.className = className; } private byte[] getRowKey() { return Bytes.toBytes(Joiner.on(':').join(APPCLASS_PREFIX, namespace.getNamespace(), className)); } } private static class PluginKey { private final Id.Namespace parentArtifactNamespace; private final String parentArtifactName; private final String type; private final String name; private PluginKey(Id.Namespace parentArtifactNamespace, String parentArtifactName, String type, String name) { this.parentArtifactNamespace = parentArtifactNamespace; this.parentArtifactName = parentArtifactName; this.type = type; this.name = name; } // p:system:etlbatch:sink:table private byte[] getRowKey() { return Bytes.toBytes(Joiner.on(':').join(PLUGIN_PREFIX, parentArtifactNamespace.getId(), parentArtifactName, type, name)); } } private static class ArtifactColumn { private final Id.Artifact artifactId; private ArtifactColumn(Id.Artifact artifactId) { this.artifactId = artifactId; } private byte[] getColumn() { return Bytes.toBytes(String.format("%s:%s:%s", artifactId.getNamespace().getId(), artifactId.getName(), artifactId.getVersion().getVersion())); } private static ArtifactColumn parse(byte[] columnBytes) { String columnStr = Bytes.toString(columnBytes); Iterator<String> parts = Splitter.on(':').limit(3).split(columnStr).iterator(); Id.Namespace namespace = Id.Namespace.from(parts.next()); return new ArtifactColumn(Id.Artifact.from(namespace, parts.next(), parts.next())); } } // utilities for creating and parsing row keys for artifacts. Keys are of the form 'r:{namespace}:{artifact-name}' private static class ArtifactKey { private final NamespaceId namespace; private final String name; private ArtifactKey(NamespaceId namespace, String name) { this.namespace = namespace; this.name = name; } private byte[] getRowKey() { return Bytes.toBytes(Joiner.on(':').join(ARTIFACT_PREFIX, namespace.getNamespace(), name)); } private static ArtifactKey parse(byte[] rowkey) { String key = Bytes.toString(rowkey); Iterator<String> parts = Splitter.on(':').limit(4).split(key).iterator(); // first part is the artifact prefix parts.next(); // next is namespace, then name return new ArtifactKey(Ids.namespace(parts.next()), parts.next()); } } private static class ArtifactCell { private final byte[] rowkey; private final byte[] column; private ArtifactCell(Id.Artifact artifactId) { rowkey = new ArtifactKey(artifactId.getNamespace().toEntityId(), artifactId.getName()).getRowKey(); column = Bytes.toBytes(artifactId.getVersion().getVersion()); } } // Data that will be stored for an artifact. Same as ArtifactDetail, expected without the id since that is redundant. private static class ArtifactData { private final URI locationURI; private final ArtifactMeta meta; public ArtifactData(Location location, ArtifactMeta meta) { this.locationURI = location.toURI(); this.meta = meta; } } // Data that will be stored for a plugin. private static class PluginData { private final PluginClass pluginClass; private final ArtifactRange usableBy; private final URI artifactLocationURI; public PluginData(PluginClass pluginClass, ArtifactRange usableBy, Location artifactLocation) { this.pluginClass = pluginClass; this.usableBy = usableBy; this.artifactLocationURI = artifactLocation.toURI(); } } // Data that will be stored for an application class. private static class AppData { private final ApplicationClass appClass; private final URI artifactLocationURI; public AppData(ApplicationClass appClass, Location artifactLocation) { this.appClass = appClass; this.artifactLocationURI = artifactLocation.toURI(); } } // serialize and deserialize artifact range private static class ArtifactRangeCodec implements JsonDeserializer<ArtifactRange>, JsonSerializer<ArtifactRange> { @Override public ArtifactRange deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { try { return ArtifactRange.parse(json.getAsString()); } catch (InvalidArtifactRangeException e) { throw new JsonParseException(e); } } @Override public JsonElement serialize(ArtifactRange src, Type typeOfSrc, JsonSerializationContext context) { return new JsonPrimitive(src.toString()); } } }