List of usage examples for org.joda.time Duration ZERO
Duration ZERO
To view the source code for org.joda.time Duration ZERO.
Click Source Link
From source file:com.google.cloud.dataflow.sdk.transforms.windowing.SlidingWindows.java
License:Apache License
/** * Assigns timestamps into half-open intervals of the form * [N * period, N * period + size), where 0 is the epoch. * * <p>If {@link SlidingWindows#every} is not called, the period defaults * to the largest time unit smaller than the given duration. For example, * specifying a size of 5 seconds will result in a default period of 1 second. *//*from w w w.j a v a 2 s . com*/ public static SlidingWindows of(Duration size) { return new SlidingWindows(getDefaultPeriod(size), size, Duration.ZERO); }
From source file:com.google.cloud.dataflow.sdk.transforms.windowing.SlidingWindows.java
License:Apache License
private SlidingWindows(Duration period, Duration size, Duration offset) { if (offset.isShorterThan(Duration.ZERO) || !offset.isShorterThan(period) || !size.isLongerThan(Duration.ZERO)) { throw new IllegalArgumentException( "SlidingWindows WindowingStrategies must have 0 <= offset < period and 0 < size"); }/*from w ww. java 2s . c om*/ this.period = period; this.size = size; this.offset = offset; }
From source file:com.google.cloud.dataflow.sdk.transforms.WithTimestamps.java
License:Apache License
/** * For a {@link SerializableFunction} {@code fn} from {@code T} to {@link Instant}, outputs a * {@link PTransform} that takes an input {@link PCollection PCollection<T>} and outputs a * {@link PCollection PCollection<T>} containing every element {@code v} in the input where * each element is output with a timestamp obtained as the result of {@code fn.apply(v)}. * * <p>If the input {@link PCollection} elements have timestamps, the output timestamp for each * element must not be before the input element's timestamp minus the value of * {@link #getAllowedTimestampSkew()}. If an output timestamp is before this time, the transform * will throw an {@link IllegalArgumentException} when executed. Use * {@link #withAllowedTimestampSkew(Duration)} to update the allowed skew. * * <p>Each output element will be in the same windows as the input element. If a new window based * on the new output timestamp is desired, apply a new instance of {@link Window#into(WindowFn)}. * * <p>This transform will fail at execution time with a {@link NullPointerException} if for any * input element the result of {@code fn.apply(v)} is {@code null}. * * <p>Example of use in Java 8:/*from w w w .java2s.com*/ * <pre>{@code * PCollection<Record> timestampedRecords = records.apply( * WithTimestamps.of((Record rec) -> rec.getInstant()); * }</pre> */ public static <T> WithTimestamps<T> of(SerializableFunction<T, Instant> fn) { return new WithTimestamps<>(fn, Duration.ZERO); }
From source file:com.google.cloud.dataflow.sdk.util.FluentBackoff.java
License:Apache License
/** * Returns a copy of this {@link FluentBackoff} that instead uses the specified initial backoff * duration.//from w w w. j ava 2 s . co m * * <p>Does not modify this object. * * @see FluentBackoff */ public FluentBackoff withInitialBackoff(Duration initialBackoff) { checkArgument(initialBackoff.isLongerThan(Duration.ZERO), "initialBackoff %s must be at least 1 millisecond", initialBackoff); return new FluentBackoff(exponent, initialBackoff, maxBackoff, maxCumulativeBackoff, maxRetries); }
From source file:com.google.cloud.dataflow.sdk.util.FluentBackoff.java
License:Apache License
/** * Returns a copy of this {@link FluentBackoff} that limits the total time spent in backoff * returned across all calls to {@link BackOff#nextBackOffMillis()}. * * <p>Does not modify this object. * * @see FluentBackoff/*from w w w .j a v a 2 s . c o m*/ */ public FluentBackoff withMaxCumulativeBackoff(Duration maxCumulativeBackoff) { checkArgument(maxCumulativeBackoff.isLongerThan(Duration.ZERO), "maxCumulativeBackoff %s must be at least 1 millisecond", maxCumulativeBackoff); return new FluentBackoff(exponent, initialBackoff, maxBackoff, maxCumulativeBackoff, maxRetries); }
From source file:com.google.cloud.dataflow.sdk.util.ReduceFnRunner.java
License:Apache License
/** * Called when an end-of-window, garbage collection, or trigger-specific timer fires. *//* ww w. j a v a 2s . co m*/ public void onTimer(TimerData timer) throws Exception { // Which window is the timer for? Preconditions.checkArgument(timer.getNamespace() instanceof WindowNamespace, "Expected timer to be in WindowNamespace, but was in %s", timer.getNamespace()); @SuppressWarnings("unchecked") WindowNamespace<W> windowNamespace = (WindowNamespace<W>) timer.getNamespace(); W window = windowNamespace.getWindow(); ReduceFn<K, InputT, OutputT, W>.Context directContext = contextFactory.base(window, StateStyle.DIRECT); ReduceFn<K, InputT, OutputT, W>.Context renamedContext = contextFactory.base(window, StateStyle.RENAMED); // Has this window had its trigger finish? // - The trigger may implement isClosed as constant false. // - If the window function does not support windowing then all windows will be considered // active. // So we must take conjunction of activeWindows and triggerRunner state. boolean windowIsActiveAndOpen = activeWindows.isActive(window) && !triggerRunner.isClosed(directContext.state()); if (!windowIsActiveAndOpen) { WindowTracing.debug("ReduceFnRunner.onTimer: Note that timer {} is for non-ACTIVE window {}", timer, window); } // If this is an end-of-window timer then we may need to set a garbage collection timer // if allowed lateness is non-zero. boolean isEndOfWindow = TimeDomain.EVENT_TIME == timer.getDomain() && timer.getTimestamp().equals(window.maxTimestamp()); // If this is a garbage collection timer then we should trigger and garbage collect the window. // We'll consider any timer at or after the end-of-window time to be a signal to garbage // collect. Instant cleanupTime = garbageCollectionTime(window); boolean isGarbageCollection = TimeDomain.EVENT_TIME == timer.getDomain() && !timer.getTimestamp().isBefore(cleanupTime); if (isGarbageCollection) { WindowTracing.debug( "ReduceFnRunner.onTimer: Cleaning up for key:{}; window:{} at {} with " + "inputWatermark:{}; outputWatermark:{}", key, window, timer.getTimestamp(), timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime()); if (windowIsActiveAndOpen) { // We need to call onTrigger to emit the final pane if required. // The final pane *may* be ON_TIME if no prior ON_TIME pane has been emitted, // and the watermark has passed the end of the window. @Nullable Instant newHold = onTrigger(directContext, renamedContext, true/* isFinished */, isEndOfWindow); Preconditions.checkState(newHold == null, "Hold placed at %s despite isFinished being true.", newHold); } // Cleanup flavor B: Clear all the remaining state for this window since we'll never // see elements for it again. clearAllState(directContext, renamedContext, windowIsActiveAndOpen); } else { WindowTracing.debug( "ReduceFnRunner.onTimer: Triggering for key:{}; window:{} at {} with " + "inputWatermark:{}; outputWatermark:{}", key, window, timer.getTimestamp(), timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime()); if (windowIsActiveAndOpen) { emitIfAppropriate(directContext, renamedContext); } if (isEndOfWindow) { // If the window strategy trigger includes a watermark trigger then at this point // there should be no data holds, either because we'd already cleared them on an // earlier onTrigger, or because we just cleared them on the above emitIfAppropriate. // We could assert this but it is very expensive. // Since we are processing an on-time firing we should schedule the garbage collection // timer. (If getAllowedLateness is zero then the timer event will be considered a // cleanup event and handled by the above). // Note we must do this even if the trigger is finished so that we are sure to cleanup // any final trigger finished bits. Preconditions.checkState(windowingStrategy.getAllowedLateness().isLongerThan(Duration.ZERO), "Unexpected zero getAllowedLateness"); WindowTracing.debug( "ReduceFnRunner.onTimer: Scheduling cleanup timer for key:{}; window:{} at {} with " + "inputWatermark:{}; outputWatermark:{}", key, directContext.window(), cleanupTime, timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime()); Preconditions.checkState(!cleanupTime.isAfter(BoundedWindow.TIMESTAMP_MAX_VALUE), "Cleanup time %s is beyond end-of-time", cleanupTime); directContext.timers().setTimer(cleanupTime, TimeDomain.EVENT_TIME); } } }
From source file:com.google.cloud.dataflow.sdk.util.WatermarkHold.java
License:Apache License
/** * Attempt to add a 'garbage collection hold' if it is required. Return the {@link Instant} at * which the hold was added (ie the end of window time plus allowed lateness), * or {@literal null} if no hold was added. * * <p>We only add the hold if it is distinct from what would be added by * {@link #addEndOfWindowHold}. In other words, {@link WindowingStrategy#getAllowedLateness} * must be non-zero.//from ww w .j av a 2 s .c om * * <p>A garbage collection hold is added in two situations: * <ol> * <li>An incoming element came in behind the output watermark, and was too late for placing * the usual element hold or an end of window hold. Place the garbage collection hold so that * we can guarantee when the pane is finally triggered its output will not be dropped due to * excessive lateness by any downstream computation. * <li>The {@link WindowingStrategy#getClosingBehavior()} is * {@link ClosingBehavior#FIRE_ALWAYS}, and thus we guarantee a final pane will be emitted * for all windows which saw at least one element. Again, the garbage collection hold guarantees * that any empty final pane can be given a timestamp which will not be considered beyond * allowed lateness by any downstream computation. * </ol> * * <p>We use {@code paneIsEmpty} to distinguish cases 1 and 2. */ @Nullable private Instant addGarbageCollectionHold(ReduceFn<?, ?, ?, W>.Context context, boolean paneIsEmpty) { Instant outputWM = timerInternals.currentOutputWatermarkTime(); Instant inputWM = timerInternals.currentInputWatermarkTime(); Instant eow = context.window().maxTimestamp(); Instant gcHold = eow.plus(windowingStrategy.getAllowedLateness()); if (!windowingStrategy.getAllowedLateness().isLongerThan(Duration.ZERO)) { WindowTracing .trace("WatermarkHold.addGarbageCollectionHold: garbage collection hold at {} is unnecessary " + "since no allowed lateness for key:{}; window:{}; inputWatermark:{}; " + "outputWatermark:{}", gcHold, context.key(), context.window(), inputWM, outputWM); return null; } if (paneIsEmpty && context.windowingStrategy().getClosingBehavior() == ClosingBehavior.FIRE_IF_NON_EMPTY) { WindowTracing .trace("WatermarkHold.addGarbageCollectionHold: garbage collection hold at {} is unnecessary " + "since empty pane and FIRE_IF_NON_EMPTY for key:{}; window:{}; inputWatermark:{}; " + "outputWatermark:{}", gcHold, context.key(), context.window(), inputWM, outputWM); return null; } Preconditions.checkState(!gcHold.isBefore(inputWM), "Garbage collection hold %s cannot be before input watermark %s", gcHold, inputWM); Preconditions.checkState(!gcHold.isAfter(BoundedWindow.TIMESTAMP_MAX_VALUE), "Garbage collection hold %s is beyond end-of-time", gcHold); // Same EXTRA_HOLD_TAG vs elementHoldTag discussion as in addEndOfWindowHold above. context.state().access(EXTRA_HOLD_TAG).add(gcHold); WindowTracing.trace( "WatermarkHold.addGarbageCollectionHold: garbage collection hold at {} is on time for " + "key:{}; window:{}; inputWatermark:{}; outputWatermark:{}", gcHold, context.key(), context.window(), inputWM, outputWM); return gcHold; }
From source file:com.google.codelabs.dataflow.LatestRides.java
License:Apache License
public static void main(String[] args) { CustomPipelineOptions options = PipelineOptionsFactory.fromArgs(args).withValidation() .as(CustomPipelineOptions.class); Pipeline p = Pipeline.create(options); p.apply(PubsubIO.Read.named("read from PubSub") .topic(String.format("projects/%s/topics/%s", options.getSourceProject(), options.getSourceTopic())) .timestampLabel("ts").withCoder(TableRowJsonCoder.of())) .apply("key rides by rideid", MapElements.via((TableRow ride) -> KV.of(ride.get("ride_id").toString(), ride)) .withOutputType(new TypeDescriptor<KV<String, TableRow>>() { }))//from w w w.ja va 2 s . com .apply("session windows on rides with early firings", Window .<KV<String, TableRow>>into(Sessions.withGapDuration(Duration.standardMinutes(60))) .triggering(AfterWatermark.pastEndOfWindow().withEarlyFirings( AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(2000)))) .accumulatingFiredPanes().withAllowedLateness(Duration.ZERO)) .apply("group ride points on same ride", Combine.perKey(new LatestPointCombine())) .apply("discard key", MapElements.via((KV<String, TableRow> a) -> a.getValue()) .withOutputType(TypeDescriptor.of(TableRow.class))) .apply(PubsubIO.Write .named("WriteToPubsub").topic(String.format("projects/%s/topics/%s", options.getSinkProject(), options.getSinkTopic())) .withCoder(TableRowJsonCoder.of())); p.run(); }
From source file:com.google.codelabs.dataflow.PickupRides.java
License:Apache License
public static void main(String[] args) { CustomPipelineOptions options = PipelineOptionsFactory.fromArgs(args).withValidation() .as(CustomPipelineOptions.class); Pipeline p = Pipeline.create(options); p.apply(PubsubIO.Read.named("read from PubSub") .topic(String.format("projects/%s/topics/%s", options.getSourceProject(), options.getSourceTopic())) .timestampLabel("ts").withCoder(TableRowJsonCoder.of())) .apply("key rides by rideid", MapElements.via((TableRow ride) -> KV.of(ride.get("ride_id").toString(), ride)) .withOutputType(new TypeDescriptor<KV<String, TableRow>>() { }))// ww w . j a v a 2 s . c om .apply("session windows on rides with early firings", Window .<KV<String, TableRow>>into(Sessions.withGapDuration(Duration.standardMinutes(1))) .triggering(AfterWatermark.pastEndOfWindow().withEarlyFirings( AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(1000)))) .accumulatingFiredPanes().withAllowedLateness(Duration.ZERO)) .apply("group ride points on same ride", Combine.perKey(new PickupPointCombine())) .apply("discard key", MapElements.via((KV<String, TableRow> a) -> a.getValue()) .withOutputType(TypeDescriptor.of(TableRow.class))) .apply("filter if no pickup", Filter.byPredicate((TableRow a) -> a.get("ride_status").equals("pickup"))) .apply(PubsubIO.Write .named("WriteToPubsub").topic(String.format("projects/%s/topics/%s", options.getSinkProject(), options.getSinkTopic())) .withCoder(TableRowJsonCoder.of())); p.run(); }
From source file:com.google.devtools.build.lib.buildeventservice.BuildEventServiceTransport.java
License:Open Source License
@Override public synchronized ListenableFuture<Void> close() { if (shutdownFuture != null) { return shutdownFuture; }/*from ww w. ja v a 2 s.c om*/ logger.log(Level.INFO, "Closing the build event service transport."); // The future is completed once the close succeeded or failed. shutdownFuture = SettableFuture.create(); uploaderExecutorService.execute(() -> { try { sendOrderedBuildEvent(besProtoUtil.streamFinished()); if (errorsReported) { // If we encountered errors before and have already reported them, then we should // not report them a second time. return; } if (bestEffortUpload) { // TODO(buchgr): The code structure currently doesn't allow to enforce a timeout for // best effort upload. if (!uploadComplete.isDone()) { report(INFO, "Asynchronous Build Event Protocol upload."); } else { Throwable uploadError = fromFuture(uploadComplete); if (uploadError != null) { report(WARNING, UPLOAD_FAILED_MESSAGE, uploadError.getMessage()); } else { report(INFO, UPLOAD_SUCCEEDED_MESSAGE); } } } else { report(INFO, "Waiting for Build Event Protocol upload to finish."); try { if (Duration.ZERO.equals(uploadTimeout)) { uploadComplete.get(); } else { uploadComplete.get(uploadTimeout.getMillis(), MILLISECONDS); } report(INFO, UPLOAD_SUCCEEDED_MESSAGE); } catch (Exception e) { uploadComplete.cancel(true); reportErrorAndFailBuild(e); } } } finally { shutdownFuture.set(null); uploaderExecutorService.shutdown(); } }); return shutdownFuture; }