org.apache.lucene.store.FSDirectory.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.lucene.store.FSDirectory.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.store;

import java.io.FileNotFoundException;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.nio.channels.ClosedChannelException; // javadoc @link
import java.nio.file.DirectoryStream;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;

import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;

/**
 * Base class for Directory implementations that store index
 * files in the file system.  
 * <a name="subclasses"></a>
 * There are currently three core
 * subclasses:
 *
 * <ul>
 *
 *  <li>{@link SimpleFSDirectory} is a straightforward
 *       implementation using Files.newByteChannel.
 *       However, it has poor concurrent performance
 *       (multiple threads will bottleneck) as it
 *       synchronizes when multiple threads read from the
 *       same file.
 *
 *  <li>{@link NIOFSDirectory} uses java.nio's
 *       FileChannel's positional io when reading to avoid
 *       synchronization when reading from the same file.
 *       Unfortunately, due to a Windows-only <a
 *       href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6265734">Sun
 *       JRE bug</a> this is a poor choice for Windows, but
 *       on all other platforms this is the preferred
 *       choice. Applications using {@link Thread#interrupt()} or
 *       {@link Future#cancel(boolean)} should use
 *       {@code RAFDirectory} instead. See {@link NIOFSDirectory} java doc
 *       for details.
 *        
 *  <li>{@link MMapDirectory} uses memory-mapped IO when
 *       reading. This is a good choice if you have plenty
 *       of virtual memory relative to your index size, eg
 *       if you are running on a 64 bit JRE, or you are
 *       running on a 32 bit JRE but your index sizes are
 *       small enough to fit into the virtual memory space.
 *       Java has currently the limitation of not being able to
 *       unmap files from user code. The files are unmapped, when GC
 *       releases the byte buffers. Due to
 *       <a href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4724038">
 *       this bug</a> in Sun's JRE, MMapDirectory's {@link IndexInput#close}
 *       is unable to close the underlying OS file handle. Only when
 *       GC finally collects the underlying objects, which could be
 *       quite some time later, will the file handle be closed.
 *       This will consume additional transient disk usage: on Windows,
 *       attempts to delete or overwrite the files will result in an
 *       exception; on other platforms, which typically have a &quot;delete on
 *       last close&quot; semantics, while such operations will succeed, the bytes
 *       are still consuming space on disk.  For many applications this
 *       limitation is not a problem (e.g. if you have plenty of disk space,
 *       and you don't rely on overwriting files on Windows) but it's still
 *       an important limitation to be aware of. This class supplies a
 *       (possibly dangerous) workaround mentioned in the bug report,
 *       which may fail on non-Sun JVMs.
 * </ul>
 *
 * <p>Unfortunately, because of system peculiarities, there is
 * no single overall best implementation.  Therefore, we've
 * added the {@link #open} method, to allow Lucene to choose
 * the best FSDirectory implementation given your
 * environment, and the known limitations of each
 * implementation.  For users who have no reason to prefer a
 * specific implementation, it's best to simply use {@link
 * #open}.  For all others, you should instantiate the
 * desired implementation directly.
 *
 * <p><b>NOTE:</b> Accessing one of the above subclasses either directly or
 * indirectly from a thread while it's interrupted can close the
 * underlying channel immediately if at the same time the thread is
 * blocked on IO. The channel will remain closed and subsequent access
 * to the index will throw a {@link ClosedChannelException}.
 * Applications using {@link Thread#interrupt()} or
 * {@link Future#cancel(boolean)} should use the slower legacy
 * {@code RAFDirectory} from the {@code misc} Lucene module instead.
 *
 * <p>The locking implementation is by default {@link
 * NativeFSLockFactory}, but can be changed by
 * passing in a custom {@link LockFactory} instance.
 *
 * @see Directory
 */
public abstract class FSDirectory extends BaseDirectory {

    protected final Path directory; // The underlying filesystem directory

    /** Maps files that we are trying to delete (or we tried already but failed)
     *  before attempting to delete that key. */
    private final Set<String> pendingDeletes = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());

    private final AtomicInteger opsSinceLastDelete = new AtomicInteger();

    /** Used to generate temp file names in {@link #createTempOutput}. */
    private final AtomicLong nextTempFileCounter = new AtomicLong();

    /** Create a new FSDirectory for the named location (ctor for subclasses).
     * The directory is created at the named location if it does not yet exist.
     * 
     * <p>{@code FSDirectory} resolves the given Path to a canonical /
     * real path to ensure it can correctly lock the index directory and no other process
     * can interfere with changing possible symlinks to the index directory inbetween.
     * If you want to use symlinks and change them dynamically, close all
     * {@code IndexWriters} and create a new {@code FSDirectory} instance.
     * @param path the path of the directory
     * @param lockFactory the lock factory to use, or null for the default
     * ({@link NativeFSLockFactory});
     * @throws IOException if there is a low-level I/O error
     */
    protected FSDirectory(Path path, LockFactory lockFactory) throws IOException {
        super(lockFactory);
        // If only read access is permitted, createDirectories fails even if the directory already exists.
        if (!Files.isDirectory(path)) {
            Files.createDirectories(path); // create directory, if it doesn't exist
        }
        directory = path.toRealPath();
    }

    /** Creates an FSDirectory instance, trying to pick the
     *  best implementation given the current environment.
     *  The directory returned uses the {@link NativeFSLockFactory}.
     *  The directory is created at the named location if it does not yet exist.
     * 
     * <p>{@code FSDirectory} resolves the given Path when calling this method to a canonical /
     * real path to ensure it can correctly lock the index directory and no other process
     * can interfere with changing possible symlinks to the index directory inbetween.
     * If you want to use symlinks and change them dynamically, close all
     * {@code IndexWriters} and create a new {@code FSDirectory} instance.
     *
     *  <p>Currently this returns {@link MMapDirectory} for Linux, MacOSX, Solaris,
     *  and Windows 64-bit JREs, {@link NIOFSDirectory} for other
     *  non-Windows JREs, and {@link SimpleFSDirectory} for other
     *  JREs on Windows. It is highly recommended that you consult the
     *  implementation's documentation for your platform before
     *  using this method.
     *
     * <p><b>NOTE</b>: this method may suddenly change which
     * implementation is returned from release to release, in
     * the event that higher performance defaults become
     * possible; if the precise implementation is important to
     * your application, please instantiate it directly,
     * instead. For optimal performance you should consider using
     * {@link MMapDirectory} on 64 bit JVMs.
     *
     * <p>See <a href="#subclasses">above</a> */
    public static FSDirectory open(Path path) throws IOException {
        return open(path, FSLockFactory.getDefault());
    }

    /** Just like {@link #open(Path)}, but allows you to
     *  also specify a custom {@link LockFactory}. */
    public static FSDirectory open(Path path, LockFactory lockFactory) throws IOException {
        if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
            return new MMapDirectory(path, lockFactory);
        } else if (Constants.WINDOWS) {
            return new SimpleFSDirectory(path, lockFactory);
        } else {
            return new NIOFSDirectory(path, lockFactory);
        }
    }

    /** Lists all files (including subdirectories) in the directory.
     *
     *  @throws IOException if there was an I/O error during listing */
    public static String[] listAll(Path dir) throws IOException {
        return listAll(dir, null);
    }

    private static String[] listAll(Path dir, Set<String> skipNames) throws IOException {
        List<String> entries = new ArrayList<>();

        try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
            for (Path path : stream) {
                String name = path.getFileName().toString();
                if (skipNames == null || skipNames.contains(name) == false) {
                    entries.add(name);
                }
            }
        }

        String[] array = entries.toArray(new String[entries.size()]);
        // Directory.listAll javadocs state that we sort the results here, so we don't let filesystem
        // specifics leak out of this abstraction:
        Arrays.sort(array);
        return array;
    }

    @Override
    public String[] listAll() throws IOException {
        ensureOpen();
        return listAll(directory, pendingDeletes);
    }

    @Override
    public long fileLength(String name) throws IOException {
        ensureOpen();
        if (pendingDeletes.contains(name)) {
            throw new NoSuchFileException("file \"" + name + "\" is pending delete");
        }
        return Files.size(directory.resolve(name));
    }

    @Override
    public IndexOutput createOutput(String name, IOContext context) throws IOException {
        ensureOpen();
        maybeDeletePendingFiles();
        // If this file was pending delete, we are now bringing it back to life:
        if (pendingDeletes.remove(name)) {
            privateDeleteFile(name, true); // try again to delete it - this is best effort
            pendingDeletes.remove(name); // watch out - if the delete fails it put
        }
        return new FSIndexOutput(name);
    }

    @Override
    public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
        ensureOpen();
        maybeDeletePendingFiles();
        while (true) {
            try {
                String name = getTempFileName(prefix, suffix, nextTempFileCounter.getAndIncrement());
                if (pendingDeletes.contains(name)) {
                    continue;
                }
                return new FSIndexOutput(name, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
            } catch (FileAlreadyExistsException faee) {
                // Retry with next incremented name
            }
        }
    }

    protected void ensureCanRead(String name) throws IOException {
        if (pendingDeletes.contains(name)) {
            throw new NoSuchFileException("file \"" + name + "\" is pending delete and cannot be opened for read");
        }
    }

    @Override
    public void sync(Collection<String> names) throws IOException {
        ensureOpen();

        for (String name : names) {
            fsync(name);
        }
        maybeDeletePendingFiles();
    }

    @Override
    public void rename(String source, String dest) throws IOException {
        ensureOpen();
        if (pendingDeletes.contains(source)) {
            throw new NoSuchFileException("file \"" + source + "\" is pending delete and cannot be moved");
        }
        maybeDeletePendingFiles();
        if (pendingDeletes.remove(dest)) {
            privateDeleteFile(dest, true); // try again to delete it - this is best effort
            pendingDeletes.remove(dest); // watch out if the delete fails it's back in here.
        }
        Files.move(directory.resolve(source), directory.resolve(dest), StandardCopyOption.ATOMIC_MOVE);
    }

    @Override
    public void syncMetaData() throws IOException {
        // TODO: to improve listCommits(), IndexFileDeleter could call this after deleting segments_Ns
        ensureOpen();
        IOUtils.fsync(directory, true);
        maybeDeletePendingFiles();
    }

    @Override
    public synchronized void close() throws IOException {
        isOpen = false;
        deletePendingFiles();
    }

    /** @return the underlying filesystem directory */
    public Path getDirectory() {
        ensureOpen();
        return directory;
    }

    @Override
    public String toString() {
        return this.getClass().getSimpleName() + "@" + directory + " lockFactory=" + lockFactory;
    }

    protected void fsync(String name) throws IOException {
        IOUtils.fsync(directory.resolve(name), false);
    }

    @Override
    public void deleteFile(String name) throws IOException {
        if (pendingDeletes.contains(name)) {
            throw new NoSuchFileException("file \"" + name + "\" is already pending delete");
        }
        privateDeleteFile(name, false);
        maybeDeletePendingFiles();
    }

    /** Try to delete any pending files that we had previously tried to delete but failed
     *  because we are on Windows and the files were still held open. */
    public synchronized void deletePendingFiles() throws IOException {
        if (pendingDeletes.isEmpty() == false) {

            // TODO: we could fix IndexInputs from FSDirectory subclasses to call this when they are closed?

            // Clone the set since we mutate it in privateDeleteFile:
            for (String name : new HashSet<>(pendingDeletes)) {
                privateDeleteFile(name, true);
            }
        }
    }

    private void maybeDeletePendingFiles() throws IOException {
        if (pendingDeletes.isEmpty() == false) {
            // This is a silly heuristic to try to avoid O(N^2), where N = number of files pending deletion, behaviour on Windows:
            int count = opsSinceLastDelete.incrementAndGet();
            if (count >= pendingDeletes.size()) {
                opsSinceLastDelete.addAndGet(-count);
                deletePendingFiles();
            }
        }
    }

    private void privateDeleteFile(String name, boolean isPendingDelete) throws IOException {
        try {
            Files.delete(directory.resolve(name));
            pendingDeletes.remove(name);
        } catch (NoSuchFileException | FileNotFoundException e) {
            // We were asked to delete a non-existent file:
            pendingDeletes.remove(name);
            if (isPendingDelete && Constants.WINDOWS) {
                // TODO: can we remove this OS-specific hacky logic?  If windows deleteFile is buggy, we should instead contain this workaround in
                // a WindowsFSDirectory ...
                // LUCENE-6684: we suppress this check for Windows, since a file could be in a confusing "pending delete" state, failing the first
                // delete attempt with access denied and then apparently falsely failing here when we try ot delete it again, with NSFE/FNFE
            } else {
                throw e;
            }
        } catch (IOException ioe) {
            // On windows, a file delete can fail because there's still an open
            // file handle against it.  We record this in pendingDeletes and
            // try again later.

            // TODO: this is hacky/lenient (we don't know which IOException this is), and
            // it should only happen on filesystems that can do this, so really we should
            // move this logic to WindowsDirectory or something

            // TODO: can/should we do if (Constants.WINDOWS) here, else throw the exc?
            // but what about a Linux box with a CIFS mount?
            pendingDeletes.add(name);
        }
    }

    final class FSIndexOutput extends OutputStreamIndexOutput {
        /**
         * The maximum chunk size is 8192 bytes, because file channel mallocs
         * a native buffer outside of stack if the write buffer size is larger.
         */
        static final int CHUNK_SIZE = 8192;

        public FSIndexOutput(String name) throws IOException {
            this(name, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
        }

        FSIndexOutput(String name, OpenOption... options) throws IOException {
            super("FSIndexOutput(path=\"" + directory.resolve(name) + "\")", name,
                    new FilterOutputStream(Files.newOutputStream(directory.resolve(name), options)) {
                        // This implementation ensures, that we never write more than CHUNK_SIZE bytes:
                        @Override
                        public void write(byte[] b, int offset, int length) throws IOException {
                            while (length > 0) {
                                final int chunk = Math.min(length, CHUNK_SIZE);
                                out.write(b, offset, chunk);
                                length -= chunk;
                                offset += chunk;
                            }
                        }
                    }, CHUNK_SIZE);
        }
    }

    @Override
    public synchronized Set<String> getPendingDeletions() throws IOException {
        deletePendingFiles();
        if (pendingDeletes.isEmpty()) {
            return Collections.emptySet();
        } else {
            return Collections.unmodifiableSet(new HashSet<>(pendingDeletes));
        }
    }
}