eu.stratosphere.nephele.io.channels.FileBufferManager.java Source code

Java tutorial

Introduction

Here is the source code for eu.stratosphere.nephele.io.channels.FileBufferManager.java

Source

/***********************************************************************************************************************
 *
 * Copyright (C) 2010 by the Stratosphere project (http://stratosphere.eu)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
 * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations under the License.
 *
 **********************************************************************************************************************/

package eu.stratosphere.nephele.io.channels;

import java.io.File;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.util.concurrent.ConcurrentHashMap;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import eu.stratosphere.nephele.checkpointing.CheckpointUtils;
import eu.stratosphere.nephele.configuration.ConfigConstants;
import eu.stratosphere.nephele.configuration.GlobalConfiguration;
import eu.stratosphere.nephele.fs.FileSystem;
import eu.stratosphere.nephele.fs.Path;
import eu.stratosphere.nephele.io.AbstractID;
import eu.stratosphere.nephele.io.channels.bytebuffered.AbstractByteBufferedInputChannel;
import eu.stratosphere.nephele.io.channels.bytebuffered.AbstractByteBufferedOutputChannel;
import eu.stratosphere.nephele.util.StringUtils;

/**
 * The file buffer manager manages the physical files which may be used to store the output or input of
 * {@link AbstractByteBufferedOutputChannel} or {@link AbstractByteBufferedInputChannel} objects, respectively. It is
 * designed as a singleton object.
 * <p>
 * This class is thread-safe.
 * 
 * @author warneke
 * @author Stephan Ewen
 */
public final class FileBufferManager {
    /**
     * The prefix with which spill files are stored.
     */
    public static final String FILE_BUFFER_PREFIX = "fb_";

    /**
     * The logging object.
     */
    private static final Log LOG = LogFactory.getLog(FileBufferManager.class);

    /**
     * The singleton instance of the file buffer manager.
     */
    private static final FileBufferManager instance = new FileBufferManager();

    /**
     * Gets the singleton instance of the file buffer manager.
     * 
     * @return the file buffer manager singleton instance
     */
    public static FileBufferManager getInstance() {
        return instance;
    }

    // --------------------------------------------------------------------------------------------

    /**
     * The map from owner IDs to files
     */
    private final ConcurrentHashMap<AbstractID, ChannelWithAccessInfo> fileMap;

    /**
     * The directories for temporary files.
     */
    private final String[] tmpDirs;

    private final int bufferSize;

    private final Path distributedTempPath;

    private final FileSystem distributedFileSystem;

    /**
     * Constructs a new file buffer manager object.
     */
    private FileBufferManager() {

        this.tmpDirs = GlobalConfiguration
                .getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH)
                .split(File.pathSeparator);

        // check temp dirs
        for (int i = 0; i < this.tmpDirs.length; i++) {
            File f = new File(this.tmpDirs[i]);
            if (!(f.exists() && f.isDirectory() && f.canWrite())) {
                LOG.error("Temp directory '" + f.getAbsolutePath() + "' is not a writable directory. "
                        + "Replacing path with default temp directory: "
                        + ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH);
                this.tmpDirs[i] = ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH;
            }
            this.tmpDirs[i] = this.tmpDirs[i] + File.separator + FILE_BUFFER_PREFIX;
        }

        this.bufferSize = GlobalConfiguration.getInteger("channel.network.bufferSizeInBytes", 64 * 1024); // TODO: Use
        // config
        // constants
        // here

        this.fileMap = new ConcurrentHashMap<AbstractID, ChannelWithAccessInfo>(2048, 0.8f, 64);

        this.distributedTempPath = CheckpointUtils.getDistributedCheckpointPath();
        FileSystem distFS = null;
        if (this.distributedTempPath != null && CheckpointUtils.allowDistributedCheckpoints()) {

            try {

                distFS = this.distributedTempPath.getFileSystem();
                if (!distFS.exists(this.distributedTempPath)) {
                    distFS.mkdirs(this.distributedTempPath);
                }

            } catch (IOException e) {
                LOG.error(StringUtils.stringifyException(e));
            }
        }

        this.distributedFileSystem = distFS;
    }

    public static boolean deleteFile(final AbstractID ownerID) {

        final FileBufferManager fbm = getInstance();
        final File f = fbm.constructLocalFile(ownerID);
        if (f.exists()) {
            f.delete();
            return true;
        }

        if (fbm.distributedTempPath != null) {
            final Path p = fbm.constructDistributedPath(ownerID);
            try {
                final FileSystem fs = p.getFileSystem();
                if (fs.exists(p)) {
                    fs.delete(p, false);
                    return true;
                }

            } catch (IOException ioe) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(StringUtils.stringifyException(ioe));
                }
            }
        }

        return false;
    }

    // --------------------------------------------------------------------------------------------

    /**
     * Gets the file channel to for the owner with the given id.
     * 
     * @param id
     *        The id for which to retrieve the channel.
     * @throws IllegalStateException
     *         Thrown, if the channel has not been registered or has already been removed.
     */
    public FileChannel getChannel(final AbstractID id, final boolean distributed) throws IOException {

        final ChannelWithAccessInfo info = getChannelInternal(id, false, distributed, false);
        if (info != null) {
            return info.getChannel();
        } else {
            throw new IllegalStateException("No channel is registered (any more) for the given id.");
        }
    }

    /**
     * Gets the file channel to for the owner with the given id and increments the references to that channel by one.
     * 
     * @param id
     *        The id for which to retrieve the channel.
     * @throws IllegalStateException
     *         Thrown, if the channel has not been registered or has already been removed.
     */
    public FileChannel getChannelAndIncrementReferences(final AbstractID owner, final boolean distributed,
            final boolean deleteOnClose) throws IOException {

        final ChannelWithAccessInfo info = getChannelInternal(owner, false, distributed, deleteOnClose);
        if (info != null) {
            return info.getAndIncrementReferences();
        } else {
            throw new IllegalStateException("No channel is registered (any more) for the given id.");
        }
    }

    /**
     * Gets the file channel to for the owner with the given id and reserved the portion of the given size for
     * writing. The position where the reserved space starts is contained in the return value. This method
     * automatically increments the number of references to the channel by one.
     * <p>
     * This method always returns a channel. If no channel exists (yet or any more) for the given id, one is created.
     * 
     * @param id
     *        The id for which to get the channel and reserve space.
     */
    public ChannelWithPosition getChannelForWriteAndIncrementReferences(final AbstractID id,
            final int spaceToReserve, final boolean distributed, final boolean deleteOnClose) throws IOException {

        ChannelWithPosition c = null;
        do {
            // the return value may be zero, if someone asynchronously decremented the counter to zero
            // and caused the disposal of the channel. falling through the loop will create a
            // new channel.
            c = getChannelInternal(id, true, distributed, deleteOnClose)
                    .reserveWriteSpaceAndIncrementReferences(spaceToReserve);

        } while (c == null);

        return c;
    }

    /**
     * Increments the references to the given channel.
     * 
     * @param id
     *        The channel to increment the references for.
     * @throws IllegalStateException
     *         Thrown, if the channel has not been registered or has already been removed.
     */
    public void incrementReferences(final AbstractID id) {

        ChannelWithAccessInfo entry = this.fileMap.get(id);
        if (entry == null || !entry.incrementReferences()) {
            throw new IllegalStateException("No channel is registered (any more) for the given id.");
        }
    }

    /**
     * Decrements the references to the given channel. If the channel reaches zero references, it will be removed.
     * 
     * @param id
     *        The channel to decrement the references for.
     * @throws IllegalStateException
     *         Thrown, if the channel has not been registered or has already been removed.
     */
    public void decrementReferences(final AbstractID id) {

        ChannelWithAccessInfo entry = this.fileMap.get(id);
        if (entry != null) {
            if (entry.decrementReferences() <= 0) {
                this.fileMap.remove(id);
            }
        } else {
            throw new IllegalStateException("Channel is not (or no longer) registered at the file buffer manager.");
        }
    }

    // --------------------------------------------------------------------------------------------

    private final Path constructDistributedPath(final AbstractID ownerID) {

        return this.distributedTempPath.suffix(Path.SEPARATOR + FILE_BUFFER_PREFIX + ownerID.toString());
    }

    private final File constructLocalFile(final AbstractID ownerID) {

        final int dirIndex = Math.abs(ownerID.hashCode()) % this.tmpDirs.length;

        return new File(this.tmpDirs[dirIndex] + ownerID.toString());
    }

    private final ChannelWithAccessInfo getChannelInternal(final AbstractID id, final boolean createIfAbsent,
            final boolean distributed, final boolean deleteOnClose) throws IOException {

        ChannelWithAccessInfo cwa = this.fileMap.get(id);
        if (cwa == null) {

            // Check if file exists
            if (distributed && this.distributedFileSystem != null) {

                final Path p = constructDistributedPath(id);

                if (this.distributedFileSystem.exists(p)) {
                    cwa = new DistributedChannelWithAccessInfo(this.distributedFileSystem, p, this.bufferSize,
                            deleteOnClose);
                }

            } else {

                final File f = constructLocalFile(id);
                if (f.exists()) {
                    cwa = new LocalChannelWithAccessInfo(f, deleteOnClose);
                }
            }

            // If file does not exist, check if we are allowed to create it
            if (createIfAbsent && cwa == null) {

                if (distributed && this.distributedFileSystem != null) {

                    final Path p = constructDistributedPath(id);
                    cwa = new DistributedChannelWithAccessInfo(this.distributedFileSystem, p, this.bufferSize,
                            deleteOnClose);

                } else {

                    // Construct the filename
                    final File f = constructLocalFile(id);
                    cwa = new LocalChannelWithAccessInfo(f, deleteOnClose);
                }
            }

            if (cwa != null) {
                final ChannelWithAccessInfo alreadyContained = this.fileMap.putIfAbsent(id, cwa);
                if (alreadyContained != null) {
                    // we had a race (should be a very rare event) and have created an
                    // unneeded channel. dispose it and use the already contained one.
                    cwa.disposeSilently();
                    cwa = alreadyContained;
                }
            } else {
                return null;
            }
        }

        cwa.updateDeleteOnCloseFlag(deleteOnClose);

        return cwa;
    }
    // --------------------------------------------------------------------------------------------

}