Java tutorial
package org.apache.samoa.utils; /* * #%L * SAMOA * %% * Copyright (C) 2014 - 2015 Apache Software Foundation * %% * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * #L% */ import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.FileSystems; import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; import org.apache.commons.io.FileUtils; import org.apache.samoa.topology.EntranceProcessingItem; import org.apache.samoa.topology.IProcessingItem; import org.apache.samoa.topology.ProcessingItem; import org.apache.samoa.topology.Stream; import org.apache.samoa.topology.impl.SamoaSystemFactory; import org.apache.samoa.topology.impl.SamzaEntranceProcessingItem; import org.apache.samoa.topology.impl.SamzaProcessingItem; import org.apache.samoa.topology.impl.SamzaStream; import org.apache.samoa.topology.impl.SamzaTopology; import org.apache.samoa.topology.impl.SamzaStream.SamzaSystemStream; import org.apache.samza.config.MapConfig; import org.apache.samza.job.local.LocalJobFactory; import org.apache.samza.job.yarn.YarnJobFactory; import org.apache.samza.system.kafka.KafkaSystemFactory; /** * Generate Configs that will be used to submit Samza jobs from the input topology (one config per PI/EntrancePI in the * topology) * * @author Anh Thu Vu * */ public class SamzaConfigFactory { public static final String SYSTEM_NAME = "samoa"; // DEFAULT VALUES private static final String DEFAULT_ZOOKEEPER = "localhost:2181"; private static final String DEFAULT_BROKER_LIST = "localhost:9092"; // DELIMINATORS public static final String COMMA = ","; public static final String COLON = ":"; public static final String DOT = "."; public static final char DOLLAR_SIGN = '$'; public static final char QUESTION_MARK = '?'; // PARTITIONING SCHEMES public static final String SHUFFLE = "shuffle"; public static final String KEY = "key"; public static final String BROADCAST = "broadcast"; // PROPERTY KEYS // JOB public static final String JOB_FACTORY_CLASS_KEY = "job.factory.class"; public static final String JOB_NAME_KEY = "job.name"; // YARN public static final String YARN_PACKAGE_KEY = "yarn.package.path"; public static final String CONTAINER_MEMORY_KEY = "yarn.container.memory.mb"; public static final String AM_MEMORY_KEY = "yarn.am.container.memory.mb"; public static final String CONTAINER_COUNT_KEY = "yarn.container.count"; // TASK (SAMZA original) public static final String TASK_CLASS_KEY = "task.class"; public static final String TASK_INPUTS_KEY = "task.inputs"; // TASK (extra) public static final String FILE_KEY = "task.processor.file"; public static final String FILESYSTEM_KEY = "task.processor.filesystem"; public static final String ENTRANCE_INPUT_KEY = "task.entrance.input"; public static final String ENTRANCE_OUTPUT_KEY = "task.entrance.outputs"; public static final String YARN_CONF_HOME_KEY = "yarn.config.home"; // KAFKA public static final String ZOOKEEPER_URI_KEY = "consumer.zookeeper.connect"; public static final String BROKER_URI_KEY = "producer.metadata.broker.list"; public static final String KAFKA_BATCHSIZE_KEY = "producer.batch.num.messages"; public static final String KAFKA_PRODUCER_TYPE_KEY = "producer.producer.type"; // SERDE public static final String SERDE_REGISTRATION_KEY = "kryo.register"; // Instance variables private boolean isLocalMode; private String zookeeper; private String kafkaBrokerList; private int replicationFactor; private int amMemory; private int containerMemory; private int piPerContainerRatio; private int checkpointFrequency; // in ms private String jarPath; private String kryoRegisterFile = null; public SamzaConfigFactory() { this.isLocalMode = false; this.zookeeper = DEFAULT_ZOOKEEPER; this.kafkaBrokerList = DEFAULT_BROKER_LIST; this.checkpointFrequency = 60000; // default: 1 minute this.replicationFactor = 1; } /* * Setter methods */ public SamzaConfigFactory setYarnPackage(String packagePath) { this.jarPath = packagePath; return this; } public SamzaConfigFactory setLocalMode(boolean isLocal) { this.isLocalMode = isLocal; return this; } public SamzaConfigFactory setZookeeper(String zk) { this.zookeeper = zk; return this; } public SamzaConfigFactory setKafka(String brokerList) { this.kafkaBrokerList = brokerList; return this; } public SamzaConfigFactory setCheckpointFrequency(int freq) { this.checkpointFrequency = freq; return this; } public SamzaConfigFactory setReplicationFactor(int replicationFactor) { this.replicationFactor = replicationFactor; return this; } public SamzaConfigFactory setAMMemory(int mem) { this.amMemory = mem; return this; } public SamzaConfigFactory setContainerMemory(int mem) { this.containerMemory = mem; return this; } public SamzaConfigFactory setPiPerContainerRatio(int piPerContainer) { this.piPerContainerRatio = piPerContainer; return this; } public SamzaConfigFactory setKryoRegisterFile(String kryoRegister) { this.kryoRegisterFile = kryoRegister; return this; } /* * Generate a map of all config properties for the input SamzaProcessingItem */ private Map<String, String> getMapForPI(SamzaProcessingItem pi, String filename, String filesystem) throws Exception { Map<String, String> map = getBasicSystemConfig(); // Set job name, task class, task inputs (from SamzaProcessingItem) setJobName(map, pi.getName()); setTaskClass(map, SamzaProcessingItem.class.getName()); StringBuilder streamNames = new StringBuilder(); boolean first = true; for (SamzaSystemStream stream : pi.getInputStreams()) { if (!first) streamNames.append(COMMA); streamNames.append(stream.getSystem() + DOT + stream.getStream()); if (first) first = false; } setTaskInputs(map, streamNames.toString()); // Processor file setFileName(map, filename); setFileSystem(map, filesystem); List<String> nameList = new ArrayList<String>(); // Default kafka system: kafka0: sync producer // This system is always required: it is used for checkpointing nameList.add("kafka0"); setKafkaSystem(map, "kafka0", this.zookeeper, this.kafkaBrokerList, 1); // Output streams: set kafka systems for (SamzaStream stream : pi.getOutputStreams()) { boolean found = false; for (String name : nameList) { if (stream.getSystemName().equals(name)) { found = true; break; } } if (!found) { nameList.add(stream.getSystemName()); setKafkaSystem(map, stream.getSystemName(), this.zookeeper, this.kafkaBrokerList, stream.getBatchSize()); } } // Input streams: set kafka systems for (SamzaSystemStream stream : pi.getInputStreams()) { boolean found = false; for (String name : nameList) { if (stream.getSystem().equals(name)) { found = true; break; } } if (!found) { nameList.add(stream.getSystem()); setKafkaSystem(map, stream.getSystem(), this.zookeeper, this.kafkaBrokerList, 1); } } // Checkpointing setValue(map, "task.checkpoint.factory", "org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory"); setValue(map, "task.checkpoint.system", "kafka0"); setValue(map, "task.commit.ms", "1000"); setValue(map, "task.checkpoint.replication.factor", Integer.toString(this.replicationFactor)); // Number of containers setNumberOfContainers(map, pi.getParallelism(), this.piPerContainerRatio); return map; } /* * Generate a map of all config properties for the input SamzaProcessingItem */ public Map<String, String> getMapForEntrancePI(SamzaEntranceProcessingItem epi, String filename, String filesystem) { Map<String, String> map = getBasicSystemConfig(); // Set job name, task class (from SamzaEntranceProcessingItem) setJobName(map, epi.getName()); setTaskClass(map, SamzaEntranceProcessingItem.class.getName()); // Input for the entrance task (from our custom consumer) setTaskInputs(map, SYSTEM_NAME + "." + epi.getName()); // Output from entrance task // Since entrancePI should have only 1 output stream // there is no need for checking the batch size, setting different system // names // The custom consumer (samoa system) does not suuport reading from a // specific index // => no need for checkpointing SamzaStream outputStream = (SamzaStream) epi.getOutputStream(); // Set samoa system factory setValue(map, "systems." + SYSTEM_NAME + ".samza.factory", SamoaSystemFactory.class.getName()); // Set Kafka system (only if there is an output stream) if (outputStream != null) setKafkaSystem(map, outputStream.getSystemName(), this.zookeeper, this.kafkaBrokerList, outputStream.getBatchSize()); // Processor file setFileName(map, filename); setFileSystem(map, filesystem); // Number of containers setNumberOfContainers(map, 1, this.piPerContainerRatio); return map; } /* * Generate a list of map (of config properties) for all PIs and EPI in the * input topology */ public List<Map<String, String>> getMapsForTopology(SamzaTopology topology) throws Exception { List<Map<String, String>> maps = new ArrayList<Map<String, String>>(); // File to write serialized objects String filename = topology.getTopologyName() + ".dat"; Path dirPath = FileSystems.getDefault().getPath("dat"); Path filePath = FileSystems.getDefault().getPath(dirPath.toString(), filename); String dstPath = filePath.toString(); String resPath; String filesystem; if (this.isLocalMode) { filesystem = SystemsUtils.LOCAL_FS; File dir = dirPath.toFile(); if (!dir.exists()) FileUtils.forceMkdir(dir); } else { filesystem = SystemsUtils.HDFS; } // Correct system name for streams this.setSystemNameForStreams(topology.getStreams()); // Add all PIs to a collection (map) Map<String, Object> piMap = new HashMap<String, Object>(); Set<EntranceProcessingItem> entranceProcessingItems = topology.getEntranceProcessingItems(); Set<IProcessingItem> processingItems = topology.getNonEntranceProcessingItems(); for (EntranceProcessingItem epi : entranceProcessingItems) { SamzaEntranceProcessingItem sepi = (SamzaEntranceProcessingItem) epi; piMap.put(sepi.getName(), sepi); } for (IProcessingItem pi : processingItems) { SamzaProcessingItem spi = (SamzaProcessingItem) pi; piMap.put(spi.getName(), spi); } // Serialize all PIs boolean serialized = false; if (this.isLocalMode) { serialized = SystemsUtils.serializeObjectToLocalFileSystem(piMap, dstPath); resPath = dstPath; } else { resPath = SystemsUtils.serializeObjectToHDFS(piMap, dstPath); serialized = resPath != null; } if (!serialized) { throw new Exception("Fail serialize map of PIs to file"); } // MapConfig for all PIs for (EntranceProcessingItem epi : entranceProcessingItems) { SamzaEntranceProcessingItem sepi = (SamzaEntranceProcessingItem) epi; maps.add(this.getMapForEntrancePI(sepi, resPath, filesystem)); } for (IProcessingItem pi : processingItems) { SamzaProcessingItem spi = (SamzaProcessingItem) pi; maps.add(this.getMapForPI(spi, resPath, filesystem)); } return maps; } /** * Construct a list of MapConfigs for a Topology * * @return the list of MapConfigs * @throws Exception */ public List<MapConfig> getMapConfigsForTopology(SamzaTopology topology) throws Exception { List<MapConfig> configs = new ArrayList<MapConfig>(); List<Map<String, String>> maps = this.getMapsForTopology(topology); for (Map<String, String> map : maps) { configs.add(new MapConfig(map)); } return configs; } /* * */ public void setSystemNameForStreams(Set<Stream> streams) { Map<Integer, String> batchSizeMap = new HashMap<Integer, String>(); batchSizeMap.put(1, "kafka0"); // default system with sync producer int counter = 0; for (Stream stream : streams) { SamzaStream samzaStream = (SamzaStream) stream; String systemName = batchSizeMap.get(samzaStream.getBatchSize()); if (systemName == null) { counter++; // Add new system systemName = "kafka" + Integer.toString(counter); batchSizeMap.put(samzaStream.getBatchSize(), systemName); } samzaStream.setSystemName(systemName); } } /* * Generate a map with common properties for PIs and EPI */ private Map<String, String> getBasicSystemConfig() { Map<String, String> map = new HashMap<String, String>(); // Job & Task if (this.isLocalMode) map.put(JOB_FACTORY_CLASS_KEY, LocalJobFactory.class.getName()); else { map.put(JOB_FACTORY_CLASS_KEY, YarnJobFactory.class.getName()); // yarn map.put(YARN_PACKAGE_KEY, jarPath); map.put(CONTAINER_MEMORY_KEY, Integer.toString(this.containerMemory)); map.put(AM_MEMORY_KEY, Integer.toString(this.amMemory)); map.put(CONTAINER_COUNT_KEY, "1"); map.put(YARN_CONF_HOME_KEY, SystemsUtils.getHadoopConfigHome()); // Task opts (Heap size = 0.75 container memory) int heapSize = (int) (0.75 * this.containerMemory); map.put("task.opts", "-Xmx" + Integer.toString(heapSize) + "M -XX:+PrintGCDateStamps"); } map.put(JOB_NAME_KEY, ""); map.put(TASK_CLASS_KEY, ""); map.put(TASK_INPUTS_KEY, ""); // register serializer map.put("serializers.registry.kryo.class", SamzaKryoSerdeFactory.class.getName()); // Serde registration setKryoRegistration(map, this.kryoRegisterFile); return map; } /* * Helper methods to set different properties in the input map */ private static void setJobName(Map<String, String> map, String jobName) { map.put(JOB_NAME_KEY, jobName); } private static void setFileName(Map<String, String> map, String filename) { map.put(FILE_KEY, filename); } private static void setFileSystem(Map<String, String> map, String filesystem) { map.put(FILESYSTEM_KEY, filesystem); } private static void setTaskClass(Map<String, String> map, String taskClass) { map.put(TASK_CLASS_KEY, taskClass); } private static void setTaskInputs(Map<String, String> map, String inputs) { map.put(TASK_INPUTS_KEY, inputs); } private static void setKryoRegistration(Map<String, String> map, String kryoRegisterFile) { if (kryoRegisterFile != null) { String value = readKryoRegistration(kryoRegisterFile); map.put(SERDE_REGISTRATION_KEY, value); } } private static void setNumberOfContainers(Map<String, String> map, int parallelism, int piPerContainer) { int res = parallelism / piPerContainer; if (parallelism % piPerContainer != 0) res++; map.put(CONTAINER_COUNT_KEY, Integer.toString(res)); } private static void setKafkaSystem(Map<String, String> map, String systemName, String zk, String brokers, int batchSize) { map.put("systems." + systemName + ".samza.factory", KafkaSystemFactory.class.getName()); map.put("systems." + systemName + ".samza.msg.serde", "kryo"); map.put("systems." + systemName + "." + ZOOKEEPER_URI_KEY, zk); map.put("systems." + systemName + "." + BROKER_URI_KEY, brokers); map.put("systems." + systemName + "." + KAFKA_BATCHSIZE_KEY, Integer.toString(batchSize)); map.put("systems." + systemName + ".samza.offset.default", "oldest"); if (batchSize > 1) { map.put("systems." + systemName + "." + KAFKA_PRODUCER_TYPE_KEY, "async"); } else { map.put("systems." + systemName + "." + KAFKA_PRODUCER_TYPE_KEY, "sync"); } } // Set custom properties private static void setValue(Map<String, String> map, String key, String value) { map.put(key, value); } /* * Helper method to parse Kryo registration file */ private static String readKryoRegistration(String filePath) { FileInputStream fis = null; Properties props = new Properties(); StringBuilder result = new StringBuilder(); try { fis = new FileInputStream(filePath); props.load(fis); boolean first = true; String value = null; for (String k : props.stringPropertyNames()) { if (!first) result.append(COMMA); else first = false; // Need to avoid the dollar sign as samza pass all the properties in // the config to containers via commandline parameters/enviroment // variables // We might escape the dollar sign, but it's more complicated than // replacing it with something else result.append(k.trim().replace(DOLLAR_SIGN, QUESTION_MARK)); value = props.getProperty(k); if (value != null && value.trim().length() > 0) { result.append(COLON); result.append(value.trim().replace(DOLLAR_SIGN, QUESTION_MARK)); } } } catch (FileNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } finally { if (fis != null) try { fis.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } return result.toString(); } }