Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.flink.streaming.connectors.fs.bucketing; import org.apache.commons.io.FileUtils; import org.apache.flink.streaming.api.operators.StreamSink; import org.apache.flink.streaming.connectors.fs.RollingSink; import org.apache.flink.streaming.connectors.fs.StringWriter; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; import org.junit.Assert; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; import java.net.URL; import java.util.List; import java.util.Map; @Deprecated public class RollingSinkMigrationTest { @ClassRule public static TemporaryFolder tempFolder = new TemporaryFolder(); private static final String PART_PREFIX = "part"; private static final String PENDING_SUFFIX = ".pending"; private static final String IN_PROGRESS_SUFFIX = ".in-progress"; private static final String VALID_LENGTH_SUFFIX = ".valid"; @Test public void testMigration() throws Exception { /* * Code ran to get the snapshot: * * final File outDir = tempFolder.newFolder(); RollingSink<String> sink = new RollingSink<String>(outDir.getAbsolutePath()) .setWriter(new StringWriter<String>()) .setBatchSize(5) .setPartPrefix(PART_PREFIX) .setInProgressPrefix("") .setPendingPrefix("") .setValidLengthPrefix("") .setInProgressSuffix(IN_PROGRESS_SUFFIX) .setPendingSuffix(PENDING_SUFFIX) .setValidLengthSuffix(VALID_LENGTH_SUFFIX); OneInputStreamOperatorTestHarness<String, Object> testHarness1 = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink)); testHarness1.setup(); testHarness1.open(); testHarness1.processElement(new StreamRecord<>("test1", 0L)); testHarness1.processElement(new StreamRecord<>("test2", 0L)); checkFs(outDir, 1, 1, 0, 0); testHarness1.processElement(new StreamRecord<>("test3", 0L)); testHarness1.processElement(new StreamRecord<>("test4", 0L)); testHarness1.processElement(new StreamRecord<>("test5", 0L)); checkFs(outDir, 1, 4, 0, 0); StreamTaskState taskState = testHarness1.snapshot(0, 0); testHarness1.snaphotToFile(taskState, "src/test/resources/rolling-sink-migration-test-flink1.1-snapshot"); testHarness1.close(); * */ final File outDir = tempFolder.newFolder(); RollingSink<String> sink = new ValidatingRollingSink<String>(outDir.getAbsolutePath()) .setWriter(new StringWriter<String>()).setBatchSize(5).setPartPrefix(PART_PREFIX) .setInProgressPrefix("").setPendingPrefix("").setValidLengthPrefix("") .setInProgressSuffix(IN_PROGRESS_SUFFIX).setPendingSuffix(PENDING_SUFFIX) .setValidLengthSuffix(VALID_LENGTH_SUFFIX); OneInputStreamOperatorTestHarness<String, Object> testHarness1 = new OneInputStreamOperatorTestHarness<>( new StreamSink<>(sink), 10, 1, 0); testHarness1.setup(); testHarness1.initializeStateFromLegacyCheckpoint( getResourceFilename("rolling-sink-migration-test-flink1.1-snapshot")); testHarness1.open(); testHarness1.processElement(new StreamRecord<>("test1", 0L)); testHarness1.processElement(new StreamRecord<>("test2", 0L)); checkFs(outDir, 1, 1, 0, 0); testHarness1.close(); } private void checkFs(File outDir, int inprogress, int pending, int completed, int valid) throws IOException { int inProg = 0; int pend = 0; int compl = 0; int val = 0; for (File file : FileUtils.listFiles(outDir, null, true)) { if (file.getAbsolutePath().endsWith("crc")) { continue; } String path = file.getPath(); if (path.endsWith(IN_PROGRESS_SUFFIX)) { inProg++; } else if (path.endsWith(PENDING_SUFFIX)) { pend++; } else if (path.endsWith(VALID_LENGTH_SUFFIX)) { val++; } else if (path.contains(PART_PREFIX)) { compl++; } } Assert.assertEquals(inprogress, inProg); Assert.assertEquals(pending, pend); Assert.assertEquals(completed, compl); Assert.assertEquals(valid, val); } private static String getResourceFilename(String filename) { ClassLoader cl = RollingSinkMigrationTest.class.getClassLoader(); URL resource = cl.getResource(filename); return resource.getFile(); } static class ValidatingRollingSink<T> extends RollingSink<T> { private static final long serialVersionUID = -4263974081712009141L; ValidatingRollingSink(String basePath) { super(basePath); } @Override public void restoreState(BucketState state) throws Exception { /** * this validates that we read the state that was checkpointed by the previous version. We expect it to be: * In-progress=/var/folders/z5/fxvg1j6s6mn94nsf8b1yc8s80000gn/T/junit2927527303216950257/junit5645682027227039270/2017-01-09--18/part-0-4 * validLength=6 * pendingForNextCheckpoint=[] * pendingForPrevCheckpoints={0=[ /var/folders/z5/fxvg1j6s6mn94nsf8b1yc8s80000gn/T/junit2927527303216950257/junit5645682027227039270/2017-01-09--18/part-0-0, * /var/folders/z5/fxvg1j6s6mn94nsf8b1yc8s80000gn/T/junit2927527303216950257/junit5645682027227039270/2017-01-09--18/part-0-1, * /var/folders/z5/fxvg1j6s6mn94nsf8b1yc8s80000gn/T/junit2927527303216950257/junit5645682027227039270/2017-01-09--18/part-0-2, * /var/folders/z5/fxvg1j6s6mn94nsf8b1yc8s80000gn/T/junit2927527303216950257/junit5645682027227039270/2017-01-09--18/part-0-3]} * */ String current = state.currentFile; long validLength = state.currentFileValidLength; Assert.assertEquals( "/var/folders/z5/fxvg1j6s6mn94nsf8b1yc8s80000gn/T/junit2927527303216950257/junit5645682027227039270/2017-01-09--18/part-0-4", current); Assert.assertEquals(6, validLength); List<String> pendingFiles = state.pendingFiles; Assert.assertTrue(pendingFiles.isEmpty()); final Map<Long, List<String>> pendingFilesPerCheckpoint = state.pendingFilesPerCheckpoint; Assert.assertEquals(1, pendingFilesPerCheckpoint.size()); for (Map.Entry<Long, List<String>> entry : pendingFilesPerCheckpoint.entrySet()) { long checkpoint = entry.getKey(); List<String> files = entry.getValue(); Assert.assertEquals(0L, checkpoint); Assert.assertEquals(4, files.size()); for (int i = 0; i < 4; i++) { Assert.assertEquals( "/var/folders/z5/fxvg1j6s6mn94nsf8b1yc8s80000gn/T/junit2927527303216950257/junit5645682027227039270/2017-01-09--18/part-0-" + i, files.get(i)); } } super.restoreState(state); } } }