org.apache.flink.runtime.fs.hdfs.HadoopRecoverableFsDataOutputStream.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.flink.runtime.fs.hdfs.HadoopRecoverableFsDataOutputStream.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.runtime.fs.hdfs;

import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.time.Deadline;
import org.apache.flink.core.fs.RecoverableFsDataOutputStream;
import org.apache.flink.core.fs.RecoverableWriter.CommitRecoverable;
import org.apache.flink.core.fs.RecoverableWriter.ResumeRecoverable;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.FlinkRuntimeException;
import org.apache.flink.util.IOUtils;
import org.apache.flink.util.Preconditions;

import org.apache.commons.lang3.time.StopWatch;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.time.Duration;

import static org.apache.flink.util.Preconditions.checkNotNull;

/**
 * An implementation of the {@link RecoverableFsDataOutputStream} for Hadoop's
 * file system abstraction.
 */
@Internal
class HadoopRecoverableFsDataOutputStream extends RecoverableFsDataOutputStream {

    private static final long LEASE_TIMEOUT = 100_000L;

    private static Method truncateHandle;

    private final FileSystem fs;

    private final Path targetFile;

    private final Path tempFile;

    private final FSDataOutputStream out;

    HadoopRecoverableFsDataOutputStream(FileSystem fs, Path targetFile, Path tempFile) throws IOException {

        ensureTruncateInitialized();

        this.fs = checkNotNull(fs);
        this.targetFile = checkNotNull(targetFile);
        this.tempFile = checkNotNull(tempFile);
        this.out = fs.create(tempFile);
    }

    HadoopRecoverableFsDataOutputStream(FileSystem fs, HadoopFsRecoverable recoverable) throws IOException {

        ensureTruncateInitialized();

        this.fs = checkNotNull(fs);
        this.targetFile = checkNotNull(recoverable.targetFile());
        this.tempFile = checkNotNull(recoverable.tempFile());

        // truncate back and append
        try {
            truncate(fs, tempFile, recoverable.offset());
        } catch (Exception e) {
            throw new IOException("Missing data in tmp file: " + tempFile, e);
        }

        waitUntilLeaseIsRevoked(tempFile);
        out = fs.append(tempFile);

        // sanity check
        long pos = out.getPos();
        if (pos != recoverable.offset()) {
            IOUtils.closeQuietly(out);
            throw new IOException(
                    "Truncate failed: " + tempFile + " (requested=" + recoverable.offset() + " ,size=" + pos + ')');
        }
    }

    @Override
    public void write(int b) throws IOException {
        out.write(b);
    }

    @Override
    public void write(byte[] b, int off, int len) throws IOException {
        out.write(b, off, len);
    }

    @Override
    public void flush() throws IOException {
        out.hflush();
    }

    @Override
    public void sync() throws IOException {
        out.hflush();
        out.hsync();
    }

    @Override
    public long getPos() throws IOException {
        return out.getPos();
    }

    @Override
    public ResumeRecoverable persist() throws IOException {
        sync();
        return new HadoopFsRecoverable(targetFile, tempFile, getPos());
    }

    @Override
    public Committer closeForCommit() throws IOException {
        final long pos = getPos();
        close();
        return new HadoopFsCommitter(fs, new HadoopFsRecoverable(targetFile, tempFile, pos));
    }

    @Override
    public void close() throws IOException {
        out.close();
    }

    // ------------------------------------------------------------------------
    //  Reflection utils for truncation
    //    These are needed to compile against Hadoop versions before
    //    Hadoop 2.7, which have no truncation calls for HDFS.
    // ------------------------------------------------------------------------

    private static void ensureTruncateInitialized() throws FlinkRuntimeException {
        if (truncateHandle == null) {
            Method truncateMethod;
            try {
                truncateMethod = FileSystem.class.getMethod("truncate", Path.class, long.class);
            } catch (NoSuchMethodException e) {
                throw new FlinkRuntimeException(
                        "Could not find a public truncate method on the Hadoop File System.");
            }

            if (!Modifier.isPublic(truncateMethod.getModifiers())) {
                throw new FlinkRuntimeException(
                        "Could not find a public truncate method on the Hadoop File System.");
            }

            truncateHandle = truncateMethod;
        }
    }

    static void truncate(FileSystem hadoopFs, Path file, long length) throws IOException {
        if (truncateHandle != null) {
            try {
                truncateHandle.invoke(hadoopFs, file, length);
            } catch (InvocationTargetException e) {
                ExceptionUtils.rethrowIOException(e.getTargetException());
            } catch (Throwable t) {
                throw new IOException(
                        "Truncation of file failed because of access/linking problems with Hadoop's truncate call. "
                                + "This is most likely a dependency conflict or class loading problem.");
            }
        } else {
            throw new IllegalStateException("Truncation handle has not been initialized");
        }
    }

    // ------------------------------------------------------------------------
    //  Committer
    // ------------------------------------------------------------------------

    /**
     * Implementation of a committer for the Hadoop File System abstraction.
     * This implementation commits by renaming the temp file to the final file path.
     * The temp file is truncated before renaming in case there is trailing garbage data.
     */
    static class HadoopFsCommitter implements Committer {

        private final FileSystem fs;
        private final HadoopFsRecoverable recoverable;

        HadoopFsCommitter(FileSystem fs, HadoopFsRecoverable recoverable) {
            this.fs = checkNotNull(fs);
            this.recoverable = checkNotNull(recoverable);
        }

        @Override
        public void commit() throws IOException {
            final Path src = recoverable.tempFile();
            final Path dest = recoverable.targetFile();
            final long expectedLength = recoverable.offset();

            final FileStatus srcStatus;
            try {
                srcStatus = fs.getFileStatus(src);
            } catch (IOException e) {
                throw new IOException("Cannot clean commit: Staging file does not exist.");
            }

            if (srcStatus.getLen() != expectedLength) {
                // something was done to this file since the committer was created.
                // this is not the "clean" case
                throw new IOException("Cannot clean commit: File has trailing junk data.");
            }

            try {
                fs.rename(src, dest);
            } catch (IOException e) {
                throw new IOException("Committing file by rename failed: " + src + " to " + dest, e);
            }
        }

        @Override
        public void commitAfterRecovery() throws IOException {
            final Path src = recoverable.tempFile();
            final Path dest = recoverable.targetFile();
            final long expectedLength = recoverable.offset();

            FileStatus srcStatus = null;
            try {
                srcStatus = fs.getFileStatus(src);
            } catch (FileNotFoundException e) {
                // status remains null
            } catch (IOException e) {
                throw new IOException("Committing during recovery failed: Could not access status of source file.");
            }

            if (srcStatus != null) {
                if (srcStatus.getLen() > expectedLength) {
                    // can happen if we go from persist to recovering for commit directly
                    // truncate the trailing junk away
                    try {
                        truncate(fs, src, expectedLength);
                    } catch (Exception e) {
                        // this can happen if the file is smaller than  expected
                        throw new IOException("Problem while truncating file: " + src, e);
                    }
                }

                // rename to final location (if it exists, overwrite it)
                try {
                    fs.rename(src, dest);
                } catch (IOException e) {
                    throw new IOException("Committing file by rename failed: " + src + " to " + dest, e);
                }
            } else if (!fs.exists(dest)) {
                // neither exists - that can be a sign of
                //   - (1) a serious problem (file system loss of data)
                //   - (2) a recovery of a savepoint that is some time old and the users
                //         removed the files in the meantime.

                // TODO how to handle this?
                // We probably need an option for users whether this should log,
                // or result in an exception or unrecoverable exception
            }
        }

        @Override
        public CommitRecoverable getRecoverable() {
            return recoverable;
        }
    }

    /**
     * Called when resuming execution after a failure and waits until the lease
     * of the file we are resuming is free.
     *
     * <p>The lease of the file we are resuming writing/committing to may still
     * belong to the process that failed previously and whose state we are
     * recovering.
     *
     * @param path The path to the file we want to resume writing to.
     */
    private boolean waitUntilLeaseIsRevoked(final Path path) throws IOException {
        Preconditions.checkState(fs instanceof DistributedFileSystem);

        final DistributedFileSystem dfs = (DistributedFileSystem) fs;
        dfs.recoverLease(path);

        final Deadline deadline = Deadline.now().plus(Duration.ofMillis(LEASE_TIMEOUT));

        final StopWatch sw = new StopWatch();
        sw.start();

        boolean isClosed = dfs.isFileClosed(path);
        while (!isClosed && deadline.hasTimeLeft()) {
            try {
                Thread.sleep(500L);
            } catch (InterruptedException e1) {
                throw new IOException("Recovering the lease failed: ", e1);
            }
            isClosed = dfs.isFileClosed(path);
        }
        return isClosed;
    }
}