com.asakusafw.runtime.directio.hadoop.DirectIoTransactionEditor.java Source code

Java tutorial

Introduction

Here is the source code for com.asakusafw.runtime.directio.hadoop.DirectIoTransactionEditor.java

Source

/**
 * Copyright 2011-2017 Asakusa Framework Team.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.asakusafw.runtime.directio.hadoop;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Scanner;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import com.asakusafw.runtime.directio.DirectDataSource;
import com.asakusafw.runtime.directio.DirectDataSourceRepository;
import com.asakusafw.runtime.directio.OutputTransactionContext;

/**
 * Edits Direct I/O transactions.
 * @since 0.2.5
 */
public final class DirectIoTransactionEditor extends Configured {

    static final Log LOG = LogFactory.getLog(DirectIoTransactionEditor.class);

    private DirectDataSourceRepository repository;

    /**
     * Creates a new instance.
     */
    public DirectIoTransactionEditor() {
        return;
    }

    /**
     * Creates a new instance.
     * @param repository repository, or {@code null} if create repository from Configuration
     */
    public DirectIoTransactionEditor(DirectDataSourceRepository repository) {
        this.repository = repository;
    }

    /**
     * Returns the corresponded transaction information to the execution ID.
     * @param executionId target ID
     * @return the corresponded transaction information, or {@code null} if does not exist
     * @throws IOException if failed to obtain information
     * @throws IllegalArgumentException if some parameters were {@code null}
     */
    public TransactionInfo get(String executionId) throws IOException {
        if (executionId == null) {
            throw new IllegalArgumentException("executionId must not be null"); //$NON-NLS-1$
        }
        Path path = HadoopDataSourceUtil.getTransactionInfoPath(getConf(), executionId);
        try {
            FileStatus status = path.getFileSystem(getConf()).getFileStatus(path);
            return toInfoObject(status);
        } catch (FileNotFoundException e) {
            return null;
        }
    }

    /**
     * Lists Direct I/O transaction information.
     * @return transactions, or an empty list if not exist
     * @throws IOException if failed to obtain transactions information
     * @throws IllegalArgumentException if some parameters were {@code null}
     */
    public List<TransactionInfo> list() throws IOException {
        LOG.info("Start listing Direct I/O transactions");
        List<FileStatus> list = new ArrayList<>(HadoopDataSourceUtil.findAllTransactionInfoFiles(getConf()));
        if (list.isEmpty()) {
            LOG.info("There are no Direct I/O transactions");
            return Collections.emptyList();
        }
        Collections.sort(list, (o1, o2) -> Long.compare(o1.getModificationTime(), o2.getModificationTime()));
        LOG.info(MessageFormat.format("Start extracting {0} Direct I/O commit information", list.size()));
        List<TransactionInfo> results = new ArrayList<>();
        for (FileStatus stat : list) {
            TransactionInfo commitObject = toInfoObject(stat);
            if (commitObject != null) {
                results.add(commitObject);
            }
        }
        LOG.info("Finish listing Direct I/O transactions");
        return results;
    }

    /**
     * Applies Direct I/O transaction for an execution.
     * @param executionId target execution ID
     * @return {@code true} if successfully applied, or {@code false} if nothing to do
     * @throws IOException if failed to apply by I/O error
     * @throws InterruptedException if interrupted
     * @throws IllegalArgumentException if some parameters were {@code null}
     */
    public boolean apply(String executionId) throws IOException, InterruptedException {
        if (executionId == null) {
            throw new IllegalArgumentException("executionId must not be null"); //$NON-NLS-1$
        }
        LOG.info(MessageFormat.format("Start applying Direct I/O transaction (executionId={0})", executionId));
        boolean applied = doApply(executionId);
        if (applied) {
            LOG.info(MessageFormat.format("Finish applying Direct I/O transaction (executionId={0})", executionId));
        } else {
            LOG.info(MessageFormat.format("Direct I/O transaction is already completed (executionId={0})",
                    executionId));
        }
        return applied;
    }

    /**
     * Aborts Direct I/O transaction for an execution.
     * @param executionId target execution ID
     * @return {@code true} if successfully aborted, or {@code false} if nothing to do
     * @throws IOException if failed to abort by I/O error
     * @throws InterruptedException if interrupted
     * @throws IllegalArgumentException if some parameters were {@code null}
     */
    public boolean abort(String executionId) throws IOException, InterruptedException {
        if (executionId == null) {
            throw new IllegalArgumentException("executionId must not be null"); //$NON-NLS-1$
        }
        LOG.info(MessageFormat.format("Start aborting Direct I/O transaction (executionId={0})", executionId));
        boolean aborted = doAbort(executionId);
        if (aborted) {
            LOG.info(MessageFormat.format("Finish aborting Direct I/O transaction (executionId={0})", executionId));
        } else {
            LOG.info(MessageFormat.format("Direct I/O transaction is already completed (executionId={0})",
                    executionId));
        }
        return aborted;
    }

    private TransactionInfo toInfoObject(FileStatus stat) throws IOException {
        assert stat != null;
        Path path = stat.getPath();
        String executionId = HadoopDataSourceUtil.getTransactionInfoExecutionId(path);
        long timestamp = stat.getModificationTime();
        List<String> comment = new ArrayList<>();
        Path commitMarkPath = HadoopDataSourceUtil.getCommitMarkPath(getConf(), executionId);
        FileSystem fs = path.getFileSystem(getConf());
        boolean committed = fs.exists(commitMarkPath);
        try (FSDataInputStream input = fs.open(path);
                Scanner scanner = new Scanner(new InputStreamReader(input, HadoopDataSourceUtil.COMMENT_CHARSET))) {
            while (scanner.hasNextLine()) {
                comment.add(scanner.nextLine());
            }
        } catch (IOException e) {
            comment.add(e.toString());
        }

        return new TransactionInfo(executionId, timestamp, committed, comment);
    }

    private boolean doApply(String executionId) throws IOException, InterruptedException {
        assert executionId != null;
        Path transactionInfo = HadoopDataSourceUtil.getTransactionInfoPath(getConf(), executionId);
        Path commitMark = HadoopDataSourceUtil.getCommitMarkPath(getConf(), executionId);
        FileSystem fs = commitMark.getFileSystem(getConf());
        if (fs.exists(transactionInfo) == false) {
            return false;
        }
        boolean succeed = true;
        if (fs.exists(commitMark) == false) {
            // FIXME cleanup
            return false;
        }
        DirectDataSourceRepository repo = getRepository();
        for (String containerPath : repo.getContainerPaths()) {
            String datasourceId = repo.getRelatedId(containerPath);
            try {
                DirectDataSource datasource = repo.getRelatedDataSource(containerPath);
                OutputTransactionContext context = HadoopDataSourceUtil.createContext(executionId, datasourceId);
                datasource.commitTransactionOutput(context);
                datasource.cleanupTransactionOutput(context);
            } catch (IOException e) {
                succeed = false;
                LOG.error(MessageFormat.format("Failed to apply transaction (datastoreId={0}, executionId={1})",
                        datasourceId, executionId));
            }
        }
        if (succeed) {
            LOG.info(MessageFormat.format("Deleting commit mark (executionId={0}, path={1})", executionId,
                    commitMark));
            try {
                if (fs.delete(commitMark, true) == false) {
                    LOG.warn(MessageFormat.format("Failed to delete commit mark (executionId={0}, path={1})",
                            executionId, commitMark));
                } else if (fs.delete(transactionInfo, true) == false) {
                    LOG.warn(MessageFormat.format("Failed to delete transaction info (executionId={0}, path={1})",
                            executionId, transactionInfo));
                }
            } catch (FileNotFoundException e) {
                LOG.warn(MessageFormat.format("Failed to delete commit mark (executionId={0}, path={1})",
                        executionId, commitMark), e);
            }
            return true;
        } else {
            throw new IOException(MessageFormat.format("Failed to apply this transaction (executionId={0});"
                    + " if you want to ignore this transaction, please abort this.", executionId));
        }
    }

    private boolean doAbort(String executionId) throws IOException, InterruptedException {
        assert executionId != null;
        Path transactionInfo = HadoopDataSourceUtil.getTransactionInfoPath(getConf(), executionId);
        Path commitMark = HadoopDataSourceUtil.getCommitMarkPath(getConf(), executionId);
        FileSystem fs = commitMark.getFileSystem(getConf());
        if (fs.exists(transactionInfo) == false) {
            return false;
        }
        boolean succeed = true;
        if (fs.exists(commitMark)) {
            LOG.info(MessageFormat.format("Deleting commit mark (executionId={0}, path={1})", executionId,
                    commitMark));
            if (fs.delete(commitMark, true) == false) {
                succeed = false;
                LOG.warn(MessageFormat.format("Failed to delete commit mark (executionId={0}, path={1})",
                        executionId, commitMark));
            }
        }
        DirectDataSourceRepository repo = getRepository();
        for (String containerPath : repo.getContainerPaths()) {
            String datasourceId = repo.getRelatedId(containerPath);
            try {
                DirectDataSource datasource = repo.getRelatedDataSource(containerPath);
                OutputTransactionContext context = HadoopDataSourceUtil.createContext(executionId, datasourceId);
                datasource.cleanupTransactionOutput(context);
            } catch (IOException e) {
                succeed = false;
                LOG.error(MessageFormat.format("Failed to abort transaction (datastoreId={0}, executionId={1})",
                        datasourceId, executionId));
            }
        }
        if (succeed) {
            LOG.info(MessageFormat.format("Deleting transaction info (executionId={0}, path={1})", executionId,
                    commitMark));
            try {
                if (fs.delete(transactionInfo, true) == false) {
                    LOG.warn(MessageFormat.format("Failed to delete transaction info (executionId={0}, path={1})",
                            executionId, transactionInfo));
                }
            } catch (FileNotFoundException e) {
                LOG.warn(MessageFormat.format("Failed to delete transaction info (executionId={0}, path={1})",
                        executionId, commitMark), e);
            }
            return true;
        } else {
            throw new IOException(MessageFormat.format(
                    "Failed to abort this transaction (executionId={0});"
                            + " if you want to ignore this transaction, please delete {1} manually.",
                    executionId, transactionInfo));
        }
    }

    private synchronized DirectDataSourceRepository getRepository() {
        if (repository == null) {
            LOG.info("Start initializing Direct I/O data stores");
            repository = HadoopDataSourceUtil.loadRepository(getConf());
            LOG.info("Finish initializing Direct I/O data stores");
        }
        return repository;
    }

    /**
     * Represents a tranaction info.
     * @since 0.2.5
     */
    public static class TransactionInfo {

        private final String executionId;

        private final long timestamp;

        private final boolean committed;

        private final List<String> comment;

        /**
         * Creates a new instance.
         * @param executionId the execution ID
         * @param timestamp timestamp
         * @param committed whether this transaction was committed
         * @param comment comment for this transaction
         * @throws IllegalArgumentException if some parameters were {@code null}
         */
        public TransactionInfo(String executionId, long timestamp, boolean committed, List<String> comment) {
            if (executionId == null) {
                throw new IllegalArgumentException("executionId must not be null"); //$NON-NLS-1$
            }
            if (comment == null) {
                throw new IllegalArgumentException("comment must not be null"); //$NON-NLS-1$
            }
            this.executionId = executionId;
            this.timestamp = timestamp;
            this.committed = committed;
            this.comment = comment;
        }

        /**
         * Returns the target execution ID.
         * @return the execution ID
         */
        public String getExecutionId() {
            return executionId;
        }

        /**
         * Returns the commit start timestamp.
         * @return the timestamp
         */
        public long getTimestamp() {
            return timestamp;
        }

        /**
         * Returns whether this transaction was committed.
         * @return the committed
         */
        public boolean isCommitted() {
            return committed;
        }

        /**
         * Returns the comment for this commit.
         * @return the comment
         */
        public List<String> getComment() {
            return comment;
        }
    }
}