Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.tinkerpop.gremlin.process.computer; import org.apache.commons.configuration.Configuration; import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.Vertex; import java.io.Serializable; import java.lang.reflect.Constructor; import java.util.Comparator; import java.util.Iterator; import java.util.Optional; /** * A MapReduce is composed of map(), combine(), and reduce() stages. * The map() stage processes the vertices of the {@link org.apache.tinkerpop.gremlin.structure.Graph} in a logically parallel manner. * The combine() stage aggregates the values of a particular map emitted key prior to sending across the cluster. * The reduce() stage aggregates the values of the combine/map emitted keys for the keys that hash to the current machine in the cluster. * The interface presented here is nearly identical to the interface popularized by Hadoop save the the map() is over the vertices of the graph. * * @author Marko A. Rodriguez (http://markorodriguez.com) */ public interface MapReduce<MK, MV, RK, RV, R> extends Cloneable { public static final String MAP_REDUCE = "gremlin.mapReduce"; /** * MapReduce is composed of three stages: map, combine, and reduce. */ public static enum Stage { MAP, COMBINE, REDUCE } /** * When it is necessary to store the state of a MapReduce job, this method is called. * This is typically required when the MapReduce job needs to be serialized to another machine. * Note that what is stored is simply the instance state, not any processed data. * * @param configuration the configuration to store the state of the MapReduce job in. */ public default void storeState(final Configuration configuration) { configuration.setProperty(MAP_REDUCE, this.getClass().getName()); } /** * When it is necessary to load the state of a MapReduce job, this method is called. * This is typically required when the MapReduce job needs to be serialized to another machine. * Note that what is loaded is simply the instance state, not any processed data. * <p/> * It is important that the state loaded from loadState() is identical to any state created from a constructor. * For those GraphComputers that do not need to use Configurations to migrate state between JVMs, the constructor will only be used. * * @param graph the graph the MapReduce job will run against * @param configuration the configuration to load the state of the MapReduce job from. */ public default void loadState(final Graph graph, final Configuration configuration) { } /** * A MapReduce job can be map-only, map-reduce-only, or map-combine-reduce. * Before executing the particular stage, this method is called to determine if the respective stage is defined. * This method should return true if the respective stage as a non-default method implementation. * * @param stage the stage to check for definition. * @return whether that stage should be executed. */ public boolean doStage(final Stage stage); /** * The map() method is logically executed at all vertices in the graph in parallel. * The map() method emits key/value pairs given some analysis of the data in the vertices (and/or its incident edges). * All {@link MapReduce} classes must at least provide an implementation of {@code MapReduce#map(Vertex, MapEmitter)}. * * @param vertex the current vertex being map() processed. * @param emitter the component that allows for key/value pairs to be emitted to the next stage. */ public void map(final Vertex vertex, final MapEmitter<MK, MV> emitter); /** * The combine() method is logically executed at all "machines" in parallel. * The combine() method pre-combines the values for a key prior to propagation over the wire. * The combine() method must emit the same key/value pairs as the reduce() method. * If there is a combine() implementation, there must be a reduce() implementation. * If the MapReduce implementation is single machine, it can skip executing this method as reduce() is sufficient. * * @param key the key that has aggregated values * @param values the aggregated values associated with the key * @param emitter the component that allows for key/value pairs to be emitted to the reduce stage. */ public default void combine(final MK key, final Iterator<MV> values, final ReduceEmitter<RK, RV> emitter) { } /** * The reduce() method is logically on the "machine" the respective key hashes to. * The reduce() method combines all the values associated with the key and emits key/value pairs. * * @param key the key that has aggregated values * @param values the aggregated values associated with the key * @param emitter the component that allows for key/value pairs to be emitted as the final result. */ public default void reduce(final MK key, final Iterator<MV> values, final ReduceEmitter<RK, RV> emitter) { } /** * This method is called at the start of the respective {@link MapReduce.Stage} for a particular "chunk of vertices." * The set of vertices in the graph are typically not processed with full parallelism. * The vertex set is split into subsets and a worker is assigned to call the MapReduce methods on it method. * The default implementation is a no-op. * * @param stage the stage of the MapReduce computation */ public default void workerStart(final Stage stage) { } /** * This method is called at the end of the respective {@link MapReduce.Stage} for a particular "chunk of vertices." * The set of vertices in the graph are typically not processed with full parallelism. * The vertex set is split into subsets and a worker is assigned to call the MapReduce methods on it method. * The default implementation is a no-op. * * @param stage the stage of the MapReduce computation */ public default void workerEnd(final Stage stage) { } /** * If a {@link Comparator} is provided, then all pairs leaving the {@link MapEmitter} are sorted. * The sorted results are either fed sorted to the combine/reduce-stage or as the final output. * If sorting is not required, then {@link Optional#empty} should be returned as sorting is computationally expensive. * The default implementation returns {@link Optional#empty}. * * @return an {@link Optional} of a comparator for sorting the map output. */ public default Optional<Comparator<MK>> getMapKeySort() { return Optional.empty(); } /** * If a {@link Comparator} is provided, then all pairs leaving the {@link ReduceEmitter} are sorted. * If sorting is not required, then {@link Optional#empty} should be returned as sorting is computationally expensive. * The default implementation returns {@link Optional#empty}. * * @return an {@link Optional} of a comparator for sorting the reduce output. */ public default Optional<Comparator<RK>> getReduceKeySort() { return Optional.empty(); } /** * The key/value pairs emitted by reduce() (or map() in a map-only job) can be iterated to generate a local JVM Java object. * * @param keyValues the key/value pairs that were emitted from reduce() (or map() in a map-only job) * @return the resultant object formed from the emitted key/values. */ public R generateFinalResult(final Iterator<KeyValue<RK, RV>> keyValues); /** * The results of the MapReduce job are associated with a memory-key to ultimately be stored in {@link Memory}. * * @return the memory key of the generated result object. */ public String getMemoryKey(); /** * The final result can be generated and added to {@link Memory} and accessible via {@link org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult}. * The default simply takes the object from generateFinalResult() and adds it to the Memory given getMemoryKey(). * * @param memory the memory of the {@link GraphComputer} * @param keyValues the key/value pairs emitted from reduce() (or map() in a map only job). */ public default void addResultToMemory(final Memory.Admin memory, final Iterator<KeyValue<RK, RV>> keyValues) { memory.set(this.getMemoryKey(), this.generateFinalResult(keyValues)); } /** * When multiple workers on a single machine need MapReduce instances, it is possible to use clone. * This will provide a speedier way of generating instances, over the {@link MapReduce#storeState} and {@link MapReduce#loadState} model. * The default implementation simply returns the object as it assumes that the MapReduce instance is a stateless singleton. * * @return A clone of the MapReduce object */ @SuppressWarnings("CloneDoesntDeclareCloneNotSupportedException") public MapReduce<MK, MV, RK, RV, R> clone(); /** * A helper method to construct a {@link MapReduce} given the content of the supplied configuration. * The class of the MapReduce is read from the {@link MapReduce#MAP_REDUCE} static configuration key. * Once the MapReduce is constructed, {@link MapReduce#loadState} method is called with the provided configuration. * * @param graph The graph that the MapReduce job will run against * @param configuration A configuration with requisite information to build a MapReduce * @return the newly constructed MapReduce */ public static <M extends MapReduce> M createMapReduce(final Graph graph, final Configuration configuration) { try { final Class<M> mapReduceClass = (Class) Class.forName(configuration.getString(MAP_REDUCE)); final Constructor<M> constructor = mapReduceClass.getDeclaredConstructor(); constructor.setAccessible(true); final M mapReduce = constructor.newInstance(); mapReduce.loadState(graph, configuration); return mapReduce; } catch (final Exception e) { throw new IllegalStateException(e.getMessage(), e); } } ////////////////// /** * The MapEmitter is used to emit key/value pairs from the map() stage of the MapReduce job. * The implementation of MapEmitter is up to the vendor, not the developer. * * @param <K> the key type * @param <V> the value type */ public interface MapEmitter<K, V> { public void emit(final K key, final V value); /** * A default method that assumes the key is {@link org.apache.tinkerpop.gremlin.process.computer.MapReduce.NullObject}. * * @param value the value to emit. */ public default void emit(final V value) { this.emit((K) MapReduce.NullObject.instance(), value); } } /** * The ReduceEmitter is used to emit key/value pairs from the combine() and reduce() stages of the MapReduce job. * The implementation of ReduceEmitter is up to the vendor, not the developer. * * @param <OK> the key type * @param <OV> the value type */ public interface ReduceEmitter<OK, OV> { public void emit(final OK key, OV value); /** * A default method that assumes the key is {@link org.apache.tinkerpop.gremlin.process.computer.MapReduce.NullObject}. * * @param value the value to emit. */ public default void emit(final OV value) { this.emit((OK) MapReduce.NullObject.instance(), value); } } ////////////////// /** * A convenience singleton when a single key is needed so that all emitted values converge to the same combiner/reducer. */ public static class NullObject implements Comparable<NullObject>, Serializable { private static final NullObject INSTANCE = new NullObject(); private static final String NULL_OBJECT = ""; public static NullObject instance() { return INSTANCE; } @Override public int hashCode() { return -9832049; } @Override public boolean equals(final Object object) { return this == object || object instanceof NullObject; } @Override public int compareTo(final NullObject object) { return 0; } @Override public String toString() { return NULL_OBJECT; } } }