Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package gobblin.service; import java.net.URI; import java.net.URISyntaxException; import java.util.Properties; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Maps; import com.google.inject.Inject; import com.google.inject.name.Named; import com.linkedin.data.template.StringMap; import com.linkedin.restli.common.ComplexResourceKey; import com.linkedin.restli.common.EmptyRecord; import com.linkedin.restli.common.HttpStatus; import com.linkedin.restli.server.CreateResponse; import com.linkedin.restli.server.RestLiServiceException; import com.linkedin.restli.server.UpdateResponse; import com.linkedin.restli.server.annotations.RestLiCollection; import com.linkedin.restli.server.resources.ComplexKeyResourceTemplate; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import gobblin.config.ConfigBuilder; import gobblin.configuration.ConfigurationKeys; import gobblin.runtime.api.FlowSpec; import gobblin.runtime.api.SpecNotFoundException; import gobblin.runtime.spec_catalog.FlowCatalog; /** * Resource for handling flow configuration requests */ @RestLiCollection(name = "flowconfigs", namespace = "gobblin.service", keyName = "id") public class FlowConfigsResource extends ComplexKeyResourceTemplate<FlowId, EmptyRecord, FlowConfig> { private static final Logger LOG = LoggerFactory.getLogger(FlowConfigsResource.class); @edu.umd.cs.findbugs.annotations.SuppressWarnings("MS_SHOULD_BE_FINAL") public static FlowCatalog _globalFlowCatalog; @Inject @Named("flowCatalog") private FlowCatalog _flowCatalog; // For blocking use of this resource until it is ready @Inject @Named("readyToUse") private Boolean readyToUse = Boolean.FALSE; public FlowConfigsResource() { } /** * Logs message and throws Rest.li exception * @param status HTTP status code * @param msg error message * @param e exception */ public void logAndThrowRestLiServiceException(HttpStatus status, String msg, Exception e) { if (e != null) { LOG.error(msg, e); throw new RestLiServiceException(status, msg + " cause = " + e.getMessage()); } else { LOG.error(msg); throw new RestLiServiceException(status, msg); } } /** * Retrieve the flow configuration with the given key * @param key flow config id key containing group name and flow name * @return {@link FlowConfig} with flow configuration */ @Override public FlowConfig get(ComplexResourceKey<FlowId, EmptyRecord> key) { String flowGroup = key.getKey().getFlowGroup(); String flowName = key.getKey().getFlowName(); LOG.info("Get called with flowGroup " + flowGroup + " flowName " + flowName); try { URI flowCatalogURI = new URI("gobblin-flow", null, "/", null, null); URI flowUri = new URI(flowCatalogURI.getScheme(), flowCatalogURI.getAuthority(), "/" + flowGroup + "/" + flowName, null, null); FlowSpec spec = (FlowSpec) getFlowCatalog().getSpec(flowUri); FlowConfig flowConfig = new FlowConfig(); Properties flowProps = spec.getConfigAsProperties(); Schedule schedule = null; if (flowProps.containsKey(ConfigurationKeys.JOB_SCHEDULE_KEY)) { schedule = new Schedule(); schedule.setCronSchedule(flowProps.getProperty(ConfigurationKeys.JOB_SCHEDULE_KEY)); } if (flowProps.containsKey(ConfigurationKeys.JOB_TEMPLATE_PATH)) { flowConfig.setTemplateUris(flowProps.getProperty(ConfigurationKeys.JOB_TEMPLATE_PATH)); } else if (spec.getTemplateURIs().isPresent()) { flowConfig.setTemplateUris(StringUtils.join(spec.getTemplateURIs().get(), ",")); } else { flowConfig.setTemplateUris("NA"); } if (schedule != null) { if (flowProps.containsKey(ConfigurationKeys.FLOW_RUN_IMMEDIATELY)) { schedule.setRunImmediately( Boolean.valueOf(flowProps.getProperty(ConfigurationKeys.FLOW_RUN_IMMEDIATELY))); } flowConfig.setSchedule(schedule); } // remove keys that were injected as part of flowSpec creation flowProps.remove(ConfigurationKeys.JOB_SCHEDULE_KEY); flowProps.remove(ConfigurationKeys.JOB_TEMPLATE_PATH); StringMap flowPropsAsStringMap = new StringMap(); flowPropsAsStringMap.putAll(Maps.fromProperties(flowProps)); return flowConfig.setId(new FlowId().setFlowGroup(flowGroup).setFlowName(flowName)) .setProperties(flowPropsAsStringMap); } catch (URISyntaxException e) { logAndThrowRestLiServiceException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowName, e); } catch (SpecNotFoundException e) { logAndThrowRestLiServiceException(HttpStatus.S_404_NOT_FOUND, "Flow requested does not exist: " + flowName, null); } return null; } /** * Build a {@link FlowSpec} from a {@link FlowConfig} * @param flowConfig flow configuration * @return {@link FlowSpec} created with attributes from flowConfig */ private FlowSpec createFlowSpecForConfig(FlowConfig flowConfig) { ConfigBuilder configBuilder = ConfigBuilder.create() .addPrimitive(ConfigurationKeys.FLOW_GROUP_KEY, flowConfig.getId().getFlowGroup()) .addPrimitive(ConfigurationKeys.FLOW_NAME_KEY, flowConfig.getId().getFlowName()); if (flowConfig.hasSchedule()) { Schedule schedule = flowConfig.getSchedule(); configBuilder.addPrimitive(ConfigurationKeys.JOB_SCHEDULE_KEY, schedule.getCronSchedule()); configBuilder.addPrimitive(ConfigurationKeys.FLOW_RUN_IMMEDIATELY, schedule.isRunImmediately()); } Config config = configBuilder.build(); Config configWithFallback = config.withFallback(ConfigFactory.parseMap(flowConfig.getProperties())); URI templateURI = null; try { templateURI = new URI(flowConfig.getTemplateUris()); } catch (URISyntaxException e) { logAndThrowRestLiServiceException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowConfig.getTemplateUris(), e); } return FlowSpec.builder().withConfig(configWithFallback).withTemplate(templateURI).build(); } /** * Create a flow configuration that the service will forward to execution instances for execution * @param flowConfig flow configuration * @return {@link CreateResponse} */ @Override public CreateResponse create(FlowConfig flowConfig) { LOG.info("Create called with flowName " + flowConfig.getId().getFlowName()); LOG.info("ReadyToUse is: " + readyToUse); LOG.info("FlowCatalog is: " + getFlowCatalog()); if (!readyToUse && getFlowCatalog() == null) { throw new RuntimeException("Not ready for use."); } try { URI flowCatalogURI = new URI("gobblin-flow", null, "/", null, null); URI flowUri = new URI(flowCatalogURI.getScheme(), flowCatalogURI.getAuthority(), "/" + flowConfig.getId().getFlowGroup() + "/" + flowConfig.getId().getFlowName(), null, null); if (getFlowCatalog().getSpec(flowUri) != null) { logAndThrowRestLiServiceException(HttpStatus.S_409_CONFLICT, "Flow with the same name already exists: " + flowUri, null); } } catch (URISyntaxException e) { logAndThrowRestLiServiceException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowConfig.getId().getFlowName(), e); } catch (SpecNotFoundException e) { // okay if flow does not exist } getFlowCatalog().put(createFlowSpecForConfig(flowConfig)); return new CreateResponse(flowConfig.getId().getFlowName(), HttpStatus.S_201_CREATED); } /** * Update the flow configuration with the specified key. Running flows are not affected. * An error is raised if the flow configuration does not exist. * @param key composite key containing group name and flow name that identifies the flow to update * @param flowConfig new flow configuration * @return {@link UpdateResponse} */ @Override public UpdateResponse update(ComplexResourceKey<FlowId, EmptyRecord> key, FlowConfig flowConfig) { String flowGroup = key.getKey().getFlowGroup(); String flowName = key.getKey().getFlowName(); URI flowUri = null; LOG.info("Update called with flowGroup " + flowGroup + " flowName " + flowName); if (!flowGroup.equals(flowConfig.getId().getFlowGroup()) || !flowName.equals(flowConfig.getId().getFlowName())) { logAndThrowRestLiServiceException(HttpStatus.S_400_BAD_REQUEST, "flowName and flowGroup cannot be changed in update", null); } try { URI flowCatalogURI = new URI("gobblin-flow", null, "/", null, null); flowUri = new URI(flowCatalogURI.getScheme(), flowCatalogURI.getAuthority(), "/" + flowGroup + "/" + flowName, null, null); FlowSpec oldFlowSpec = (FlowSpec) getFlowCatalog().getSpec(flowUri); FlowSpec newFlowSpec = createFlowSpecForConfig(flowConfig); getFlowCatalog().put(newFlowSpec); return new UpdateResponse(HttpStatus.S_200_OK); } catch (URISyntaxException e) { logAndThrowRestLiServiceException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowUri, e); } catch (SpecNotFoundException e) { logAndThrowRestLiServiceException(HttpStatus.S_404_NOT_FOUND, "Flow does not exist: flowGroup " + flowGroup + " flowName " + flowName, null); } return null; } /** * Delete a configured flow. Running flows are not affected. The schedule will be removed for scheduled flows. * @param key composite key containing flow group and flow name that identifies the flow to remove from the * {@link FlowCatalog} * @return {@link UpdateResponse} */ @Override public UpdateResponse delete(ComplexResourceKey<FlowId, EmptyRecord> key) { String flowGroup = key.getKey().getFlowGroup(); String flowName = key.getKey().getFlowName(); URI flowUri = null; LOG.info("Delete called with flowGroup " + flowGroup + " flowName " + flowName); try { URI flowCatalogURI = new URI("gobblin-flow", null, "/", null, null); flowUri = new URI(flowCatalogURI.getScheme(), flowCatalogURI.getAuthority(), "/" + flowGroup + "/" + flowName, null, null); FlowSpec flowSpec = (FlowSpec) getFlowCatalog().getSpec(flowUri); getFlowCatalog().remove(flowUri); return new UpdateResponse(HttpStatus.S_200_OK); } catch (URISyntaxException e) { logAndThrowRestLiServiceException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowUri, e); } catch (SpecNotFoundException e) { logAndThrowRestLiServiceException(HttpStatus.S_404_NOT_FOUND, "Flow does not exist: flowGroup " + flowGroup + " flowName " + flowName, null); } return null; } /*** * This method is to workaround injection issues where Service has only one active global FlowCatalog * .. and is not able to inject it via RestLI bootstrap. We should remove this and make injected * .. FlowCatalog standard after injection works and recipe is documented here. * @return FlowCatalog in use. */ private FlowCatalog getFlowCatalog() { if (null != _globalFlowCatalog) { return _globalFlowCatalog; } return this._flowCatalog; } }