Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.beam.sdk.transforms.windowing; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.Objects; import org.apache.beam.sdk.coders.AtomicCoder; import org.apache.beam.sdk.coders.CoderException; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.DoFn.WindowedContext; import org.apache.beam.sdk.transforms.GroupByKey; import org.apache.beam.sdk.util.VarInt; /** * Provides information about the pane an element belongs to. Every pane is implicitly associated * with a window. Panes are observable only via the * {@link DoFn.ProcessContext#pane} method of the context * passed to a {@link DoFn.ProcessElement} method. * * <p>Note: This does not uniquely identify a pane, and should not be used for comparisons. */ public final class PaneInfo { /** * Enumerates the possibilities for the timing of this pane firing related to the * input and output watermarks for its computation. * * <p>A window may fire multiple panes, and the timing of those panes generally follows the * regular expression {@code EARLY* ON_TIME? LATE*}. Generally a pane is considered: * <ol> * <li>{@code EARLY} if the system cannot be sure it has seen all data which may contribute * to the pane's window. * <li>{@code ON_TIME} if the system predicts it has seen all the data which may contribute * to the pane's window. * <li>{@code LATE} if the system has encountered new data after predicting no more could arrive. * It is possible an {@code ON_TIME} pane has already been emitted, in which case any * following panes are considered {@code LATE}. * </ol> * * <p>Only an * {@link AfterWatermark#pastEndOfWindow} trigger may produce an {@code ON_TIME} pane. * With merging {@link WindowFn}'s, windows may be merged to produce new windows that satisfy * their own instance of the above regular expression. The only guarantee is that once a window * produces a final pane, it will not be merged into any new windows. * * <p>The predictions above are made using the mechanism of watermarks. * * <p>We can state some properties of {@code LATE} and {@code ON_TIME} panes, but first need some * definitions: * <ol> * <li>We'll call a pipeline 'simple' if it does not use * {@link WindowedContext#outputWithTimestamp} in * any {@link DoFn}, and it uses the same * {@link org.apache.beam.sdk.transforms.windowing.Window#withAllowedLateness} * argument value on all windows (or uses the default of {@link org.joda.time.Duration#ZERO}). * <li>We'll call an element 'locally late', from the point of view of a computation on a * worker, if the element's timestamp is before the input watermark for that computation * on that worker. The element is otherwise 'locally on-time'. * <li>We'll say 'the pane's timestamp' to mean the timestamp of the element produced to * represent the pane's contents. * </ol> * * <p>Then in simple pipelines: * <ol> * <li> (Soundness) An {@code ON_TIME} pane can never cause a later computation to generate a * {@code LATE} pane. (If it did, it would imply a later computation's input watermark progressed * ahead of an earlier stage's output watermark, which by design is not possible.) * <li> (Liveness) An {@code ON_TIME} pane is emitted as soon as possible after the input * watermark passes the end of the pane's window. * <li> (Consistency) A pane with only locally on-time elements will always be {@code ON_TIME}. * And a {@code LATE} pane cannot contain locally on-time elements. * </ol> * * <p>However, note that: * <ol> * <li> An {@code ON_TIME} pane may contain locally late elements. It may even contain only * locally late elements. Provided a locally late element finds its way into an {@code ON_TIME} * pane its lateness becomes unobservable. * <li> A {@code LATE} pane does not necessarily cause any following computation panes to be * marked as {@code LATE}. * </ol> */ public enum Timing { /** * Pane was fired before the input watermark had progressed after the end of the window. */ EARLY, /** * Pane was fired by a {@link AfterWatermark#pastEndOfWindow} trigger because the input * watermark progressed after the end of the window. However the output watermark has not * yet progressed after the end of the window. Thus it is still possible to assign a timestamp * to the element representing this pane which cannot be considered locally late by any * following computation. */ ON_TIME, /** * Pane was fired after the output watermark had progressed past the end of the window. */ LATE, /** * This element was not produced in a triggered pane and its relation to input and * output watermarks is unknown. */ UNKNOWN // NOTE: Do not add fields or re-order them. The ordinal is used as part of // the encoding. } private static byte encodedByte(boolean isFirst, boolean isLast, Timing timing) { byte result = 0x0; if (isFirst) { result |= 1; } if (isLast) { result |= 2; } result |= timing.ordinal() << 2; return result; } private static final ImmutableMap<Byte, PaneInfo> BYTE_TO_PANE_INFO; static { ImmutableMap.Builder<Byte, PaneInfo> decodingBuilder = ImmutableMap.builder(); for (Timing timing : Timing.values()) { long onTimeIndex = timing == Timing.EARLY ? -1 : 0; register(decodingBuilder, new PaneInfo(true, true, timing, 0, onTimeIndex)); register(decodingBuilder, new PaneInfo(true, false, timing, 0, onTimeIndex)); register(decodingBuilder, new PaneInfo(false, true, timing, -1, onTimeIndex)); register(decodingBuilder, new PaneInfo(false, false, timing, -1, onTimeIndex)); } BYTE_TO_PANE_INFO = decodingBuilder.build(); } private static void register(ImmutableMap.Builder<Byte, PaneInfo> builder, PaneInfo info) { builder.put(info.encodedByte, info); } private final byte encodedByte; private final boolean isFirst; private final boolean isLast; private final Timing timing; private final long index; private final long nonSpeculativeIndex; /** * {@code PaneInfo} to use for elements on (and before) initial window assignemnt (including * elements read from sources) before they have passed through a {@link GroupByKey} and are * associated with a particular trigger firing. */ public static final PaneInfo NO_FIRING = PaneInfo.createPane(true, true, Timing.UNKNOWN, 0, 0); /** * {@code PaneInfo} to use when there will be exactly one firing and it is on time. */ public static final PaneInfo ON_TIME_AND_ONLY_FIRING = PaneInfo.createPane(true, true, Timing.ON_TIME, 0, 0); private PaneInfo(boolean isFirst, boolean isLast, Timing timing, long index, long onTimeIndex) { this.encodedByte = encodedByte(isFirst, isLast, timing); this.isFirst = isFirst; this.isLast = isLast; this.timing = timing; this.index = index; this.nonSpeculativeIndex = onTimeIndex; } public static PaneInfo createPane(boolean isFirst, boolean isLast, Timing timing) { checkArgument(isFirst, "Indices must be provided for non-first pane info."); return createPane(isFirst, isLast, timing, 0, timing == Timing.EARLY ? -1 : 0); } /** * Factory method to create a {@link PaneInfo} with the specified parameters. */ public static PaneInfo createPane(boolean isFirst, boolean isLast, Timing timing, long index, long onTimeIndex) { if (isFirst || timing == Timing.UNKNOWN) { return checkNotNull(BYTE_TO_PANE_INFO.get(encodedByte(isFirst, isLast, timing))); } else { return new PaneInfo(isFirst, isLast, timing, index, onTimeIndex); } } public static PaneInfo decodePane(byte encodedPane) { return checkNotNull(BYTE_TO_PANE_INFO.get(encodedPane)); } /** * Return true if there is no timing information for the current {@link PaneInfo}. * This typically indicates that the current element has not been assigned to * windows or passed through an operation that executes triggers yet. */ public boolean isUnknown() { return Timing.UNKNOWN.equals(timing); } /** * Return true if this is the first pane produced for the associated window. */ public boolean isFirst() { return isFirst; } /** * Return true if this is the last pane that will be produced in the associated window. */ public boolean isLast() { return isLast; } /** * Return the timing of this pane. */ public Timing getTiming() { return timing; } /** * The zero-based index of this trigger firing that produced this pane. * * <p>This will return 0 for the first time the timer fires, 1 for the next time, etc. * * <p>A given (key, window, pane-index) is guaranteed to be unique in the * output of a group-by-key operation. */ public long getIndex() { return index; } /** * The zero-based index of this trigger firing among non-speculative panes. * * <p>This will return 0 for the first non-{@link Timing#EARLY} timer firing, 1 for the next one, * etc. * * <p>Always -1 for speculative data. */ public long getNonSpeculativeIndex() { return nonSpeculativeIndex; } int getEncodedByte() { return encodedByte; } @Override public int hashCode() { return Objects.hash(encodedByte, index, nonSpeculativeIndex); } @Override public boolean equals(Object obj) { if (this == obj) { // Simple PaneInfos are interned. return true; } else if (obj instanceof PaneInfo) { PaneInfo that = (PaneInfo) obj; return this.encodedByte == that.encodedByte && this.index == that.index && this.nonSpeculativeIndex == that.nonSpeculativeIndex; } else { return false; } } @Override public String toString() { if (this == PaneInfo.NO_FIRING) { return "PaneInfo.NO_FIRING"; } return MoreObjects.toStringHelper(getClass()).omitNullValues().add("isFirst", isFirst ? true : null) .add("isLast", isLast ? true : null).add("timing", timing).add("index", index) .add("onTimeIndex", nonSpeculativeIndex != -1 ? nonSpeculativeIndex : null).toString(); } /** * A Coder for encoding PaneInfo instances. */ public static class PaneInfoCoder extends AtomicCoder<PaneInfo> { private enum Encoding { FIRST, ONE_INDEX, TWO_INDICES; // NOTE: Do not reorder fields. The ordinal is used as part of // the encoding. public final byte tag; Encoding() { assert ordinal() < 16; tag = (byte) (ordinal() << 4); } public static Encoding fromTag(byte b) { return Encoding.values()[b >> 4]; } } private Encoding chooseEncoding(PaneInfo value) { if (value.index == 0 && value.nonSpeculativeIndex == 0 || value.timing == Timing.UNKNOWN) { return Encoding.FIRST; } else if (value.index == value.nonSpeculativeIndex || value.timing == Timing.EARLY) { return Encoding.ONE_INDEX; } else { return Encoding.TWO_INDICES; } } public static final PaneInfoCoder INSTANCE = new PaneInfoCoder(); public static PaneInfoCoder of() { return INSTANCE; } private PaneInfoCoder() { } @Override public void encode(PaneInfo value, final OutputStream outStream) throws CoderException, IOException { Encoding encoding = chooseEncoding(value); switch (chooseEncoding(value)) { case FIRST: outStream.write(value.encodedByte); break; case ONE_INDEX: outStream.write(value.encodedByte | encoding.tag); VarInt.encode(value.index, outStream); break; case TWO_INDICES: outStream.write(value.encodedByte | encoding.tag); VarInt.encode(value.index, outStream); VarInt.encode(value.nonSpeculativeIndex, outStream); break; default: throw new CoderException("Unknown encoding " + encoding); } } @Override public PaneInfo decode(final InputStream inStream) throws CoderException, IOException { byte keyAndTag = (byte) inStream.read(); PaneInfo base = BYTE_TO_PANE_INFO.get((byte) (keyAndTag & 0x0F)); long index, onTimeIndex; switch (Encoding.fromTag(keyAndTag)) { case FIRST: return base; case ONE_INDEX: index = VarInt.decodeLong(inStream); onTimeIndex = base.timing == Timing.EARLY ? -1 : index; break; case TWO_INDICES: index = VarInt.decodeLong(inStream); onTimeIndex = VarInt.decodeLong(inStream); break; default: throw new CoderException("Unknown encoding " + (keyAndTag & 0xF0)); } return new PaneInfo(base.isFirst, base.isLast, base.timing, index, onTimeIndex); } @Override public void verifyDeterministic() { } } }