Java tutorial
/* * Copyright 2015 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package co.cask.cdap.internal.app.services; import co.cask.cdap.api.ProgramSpecification; import co.cask.cdap.api.flow.FlowSpecification; import co.cask.cdap.api.flow.FlowletConnection; import co.cask.cdap.api.metrics.MetricDeleteQuery; import co.cask.cdap.api.metrics.MetricStore; import co.cask.cdap.api.schedule.SchedulableProgramType; import co.cask.cdap.api.workflow.WorkflowSpecification; import co.cask.cdap.app.ApplicationSpecification; import co.cask.cdap.app.program.Programs; import co.cask.cdap.app.runtime.ProgramRuntimeService; import co.cask.cdap.app.store.Store; import co.cask.cdap.common.CannotBeDeletedException; import co.cask.cdap.common.NotFoundException; import co.cask.cdap.common.conf.CConfiguration; import co.cask.cdap.common.conf.Constants; import co.cask.cdap.common.namespace.NamespacedLocationFactory; import co.cask.cdap.config.PreferencesStore; import co.cask.cdap.data2.registry.UsageRegistry; import co.cask.cdap.data2.transaction.queue.QueueAdmin; import co.cask.cdap.data2.transaction.stream.StreamConsumerFactory; import co.cask.cdap.gateway.handlers.AppLifecycleHttpHandler; import co.cask.cdap.internal.app.runtime.flow.FlowUtils; import co.cask.cdap.internal.app.runtime.schedule.Scheduler; import co.cask.cdap.proto.Id; import co.cask.cdap.proto.ProgramType; import co.cask.cdap.proto.ProgramTypes; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.HashMultimap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.util.concurrent.AbstractIdleService; import com.google.inject.Inject; import org.apache.twill.filesystem.Location; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; /** * Service that manage lifecycle of Applications * TODO: Currently this only handles the deletion of the application. The code from {@link AppLifecycleHttpHandler} * should be moved here and the calls should be delegated to this class. */ public class ApplicationLifecycleService extends AbstractIdleService { private static final Logger LOG = LoggerFactory.getLogger(ApplicationLifecycleService.class); /** * Runtime program service for running and managing programs. */ private final ProgramRuntimeService runtimeService; /** * Store manages non-runtime lifecycle. */ private final Store store; private final CConfiguration configuration; private final Scheduler scheduler; private final QueueAdmin queueAdmin; private final NamespacedLocationFactory namespacedLocationFactory; private final StreamConsumerFactory streamConsumerFactory; private final UsageRegistry usageRegistry; private final PreferencesStore preferencesStore; private final MetricStore metricStore; @Inject public ApplicationLifecycleService(ProgramRuntimeService runtimeService, Store store, CConfiguration configuration, Scheduler scheduler, QueueAdmin queueAdmin, NamespacedLocationFactory namespacedLocationFactory, StreamConsumerFactory streamConsumerFactory, UsageRegistry usageRegistry, PreferencesStore preferencesStore, MetricStore metricStore) { this.runtimeService = runtimeService; this.store = store; this.configuration = configuration; this.scheduler = scheduler; this.queueAdmin = queueAdmin; this.namespacedLocationFactory = namespacedLocationFactory; this.streamConsumerFactory = streamConsumerFactory; this.usageRegistry = usageRegistry; this.preferencesStore = preferencesStore; this.metricStore = metricStore; } @Override protected void startUp() throws Exception { LOG.info("Starting ApplicationLifecycleService"); } @Override protected void shutDown() throws Exception { LOG.info("Shutting down ApplicationLifecycleService"); } /** * Remove all the applications inside the given {@link Id.Namespace} * * @param identifier the {@link Id.Namespace} under which all application should be deleted * @throws Exception */ public void removeAll(Id.Namespace identifier) throws Exception { List<ApplicationSpecification> allSpecs = new ArrayList<>(store.getAllApplications(identifier)); //Check if any program associated with this namespace is running final Id.Namespace accId = Id.Namespace.from(identifier.getId()); boolean appRunning = runtimeService.checkAnyRunning(new Predicate<Id.Program>() { @Override public boolean apply(Id.Program programId) { return programId.getApplication().getNamespace().equals(accId); } }, ProgramType.values()); if (appRunning) { throw new CannotBeDeletedException(identifier, "One of the program associated with this namespace is still " + "running"); } //All Apps are STOPPED, delete them for (ApplicationSpecification appSpec : allSpecs) { Id.Application id = Id.Application.from(identifier.getId(), appSpec.getName()); removeApplication(id); } } /** * Delete an application specified by appId. * * @param appId the {@link Id.Application} of the application to be removed * @throws Exception */ public void removeApplication(final Id.Application appId) throws Exception { //Check if all are stopped. boolean appRunning = runtimeService.checkAnyRunning(new Predicate<Id.Program>() { @Override public boolean apply(Id.Program programId) { return programId.getApplication().equals(appId); } }, ProgramType.values()); if (appRunning) { throw new CannotBeDeletedException(appId); } ApplicationSpecification spec = store.getApplication(appId); if (spec == null) { throw new NotFoundException(appId); } //Delete the schedules for (WorkflowSpecification workflowSpec : spec.getWorkflows().values()) { Id.Program workflowProgramId = Id.Program.from(appId, ProgramType.WORKFLOW, workflowSpec.getName()); scheduler.deleteSchedules(workflowProgramId, SchedulableProgramType.WORKFLOW); } deleteMetrics(appId.getNamespaceId(), appId.getId()); //Delete all preferences of the application and of all its programs deletePreferences(appId); // Delete all streams and queues state of each flow // TODO: This should be unified with the DeletedProgramHandlerStage for (FlowSpecification flowSpecification : spec.getFlows().values()) { Id.Program flowProgramId = Id.Program.from(appId, ProgramType.FLOW, flowSpecification.getName()); // Collects stream name to all group ids consuming that stream Multimap<String, Long> streamGroups = HashMultimap.create(); for (FlowletConnection connection : flowSpecification.getConnections()) { if (connection.getSourceType() == FlowletConnection.Type.STREAM) { long groupId = FlowUtils.generateConsumerGroupId(flowProgramId, connection.getTargetName()); streamGroups.put(connection.getSourceName(), groupId); } } // Remove all process states and group states for each stream String namespace = String.format("%s.%s", flowProgramId.getApplicationId(), flowProgramId.getId()); for (Map.Entry<String, Collection<Long>> entry : streamGroups.asMap().entrySet()) { streamConsumerFactory.dropAll(Id.Stream.from(appId.getNamespaceId(), entry.getKey()), namespace, entry.getValue()); } queueAdmin.dropAllForFlow(Id.Flow.from(appId, flowSpecification.getName())); } deleteProgramLocations(appId); Location appArchive = store.getApplicationArchiveLocation(appId); Preconditions.checkNotNull(appArchive, "Could not find the location of application", appId.getId()); if (!appArchive.delete()) { LOG.debug("Could not delete application archive"); } store.removeApplication(appId); try { usageRegistry.unregister(appId); } catch (Exception e) { LOG.warn("Failed to unregister usage of app: {}", appId, e); } } /** * Delete the jar location of the program. * * @param appId applicationId. * @throws IOException if there are errors with location IO */ private void deleteProgramLocations(Id.Application appId) throws IOException { Iterable<ProgramSpecification> programSpecs = getProgramSpecs(appId); String appFabricDir = configuration.get(Constants.AppFabric.OUTPUT_DIR); for (ProgramSpecification spec : programSpecs) { ProgramType type = ProgramTypes.fromSpecification(spec); Id.Program programId = Id.Program.from(appId, type, spec.getName()); try { Location location = Programs.programLocation(namespacedLocationFactory, appFabricDir, programId); location.delete(); } catch (FileNotFoundException e) { LOG.warn("Program jar for program {} not found.", programId.toString(), e); } } // Delete webapp // TODO: this will go away once webapp gets a spec try { Id.Program programId = Id.Program.from(appId.getNamespaceId(), appId.getId(), ProgramType.WEBAPP, ProgramType.WEBAPP.name().toLowerCase()); Location location = Programs.programLocation(namespacedLocationFactory, appFabricDir, programId); location.delete(); } catch (FileNotFoundException e) { // expected exception when webapp is not present. } } private Iterable<ProgramSpecification> getProgramSpecs(Id.Application appId) { ApplicationSpecification appSpec = store.getApplication(appId); return Iterables.concat(appSpec.getFlows().values(), appSpec.getMapReduce().values(), appSpec.getServices().values(), appSpec.getSpark().values(), appSpec.getWorkers().values(), appSpec.getWorkflows().values()); } /** * Delete the metrics for an application, or if null is provided as the application ID, for all apps. * * @param applicationId the application to delete metrics for. * If null, metrics for all applications in the namespace are deleted. */ private void deleteMetrics(String namespaceId, String applicationId) throws Exception { Collection<ApplicationSpecification> applications = Lists.newArrayList(); if (applicationId == null) { applications = this.store.getAllApplications(new Id.Namespace(namespaceId)); } else { ApplicationSpecification spec = this.store .getApplication(new Id.Application(new Id.Namespace(namespaceId), applicationId)); applications.add(spec); } long endTs = System.currentTimeMillis() / 1000; Map<String, String> tags = Maps.newHashMap(); tags.put(Constants.Metrics.Tag.NAMESPACE, namespaceId); for (ApplicationSpecification application : applications) { // add or replace application name in the tagMap tags.put(Constants.Metrics.Tag.APP, application.getName()); MetricDeleteQuery deleteQuery = new MetricDeleteQuery(0, endTs, tags); metricStore.delete(deleteQuery); } } /** * Delete stored Preferences of the application and all its programs. * * @param appId applicationId */ private void deletePreferences(Id.Application appId) { Iterable<ProgramSpecification> programSpecs = getProgramSpecs(appId); for (ProgramSpecification spec : programSpecs) { preferencesStore.deleteProperties(appId.getNamespaceId(), appId.getId(), ProgramTypes.fromSpecification(spec).getCategoryName(), spec.getName()); LOG.trace("Deleted Preferences of Program : {}, {}, {}, {}", appId.getNamespaceId(), appId.getId(), ProgramTypes.fromSpecification(spec).getCategoryName(), spec.getName()); } preferencesStore.deleteProperties(appId.getNamespaceId(), appId.getId()); LOG.trace("Deleted Preferences of Application : {}, {}", appId.getNamespaceId(), appId.getId()); } }