org.sakaiproject.search.index.impl.JDBCClusterIndexStore.java Source code

Java tutorial

Introduction

Here is the source code for org.sakaiproject.search.index.impl.JDBCClusterIndexStore.java

Source

/**********************************************************************************
 * $URL: https://source.sakaiproject.org/svn/search/trunk/search-impl/impl/src/java/org/sakaiproject/search/index/impl/JDBCClusterIndexStore.java $
 * $Id: JDBCClusterIndexStore.java 111640 2012-08-20 12:58:11Z david.horwitz@uct.ac.za $
 ***********************************************************************************
 *
 * Copyright (c) 2003, 2004, 2005, 2006, 2007, 2008 The Sakai Foundation
 *
 * Licensed under the Educational Community License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.opensource.org/licenses/ECL-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 **********************************************************************************/

package org.sakaiproject.search.index.impl;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.channels.FileChannel;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;

import javax.sql.DataSource;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.store.FSDirectory;
import org.sakaiproject.search.api.SearchService;
import org.sakaiproject.search.index.ClusterFilesystem;
import org.sakaiproject.search.index.SegmentInfo;

/**
 * This is a JDBC implementation of the ClusterFilesystem. It syncronizes the
 * local search index segments with the database, by sipping each segment and
 * pushing it to the database. Each Segment has an extra file that contains an
 * MD5 of the segment and a time stamp of the last update. If any segments are
 * missing in the local segment store (including no segments at all, as with a
 * new cluster node) the missing segments are downloaded from the JDBC store. If
 * any of the segments on the local store are found to be dammaged they are
 * reloaded from the database.
 * 
 * @author ieb
 */
public class JDBCClusterIndexStore implements ClusterFilesystem {

    private static final Log log = LogFactory.getLog(JDBCClusterIndexStore.class);

    private DataSource dataSource = null;

    private String searchIndexDirectory = null;

    private static final String TEMP_INDEX_NAME = "tempindex";

    private static final String INDEX_PATCHNAME = "indexpatch";

    private static final long MAX_BLOCK_SIZE = 1024 * 1024 * 10; // 10M
    // blocks

    private boolean autoDdl = false;

    private boolean parallelIndex = false;

    /**
     * If validate is true, all segments will be checked on initial startup and
     * upload. This can take a long time. If its false, only when an index is
     * updated is the MD5 checked. Recomendation is to leave this false.
     */
    private boolean validate = false;

    private String sharedSegments = null;

    private boolean debug = false;

    /**
     * locatStructuredStorage causes local segments to be placed into structured
     * storage on the local disk
     */
    private boolean localStructuredStorage = false;

    /**
     * sharedStructuredStorage causes the shared segments to be placed into
     * shared structured storage in the shared location
     */
    private boolean sharedStructuredStorage = false;

    private ClusterSegmentsStorage clusterStorage = null;

    private boolean localSegmentsOnly = false;

    private SearchService searchService;

    public void init() {
        try {
            log.info(this + ":init() ");
            clusterStorage = new ClusterSegmentsStorage(searchService, searchIndexDirectory, this,
                    localStructuredStorage, debug);

            // We should migrate to the correct storage format, on the
            // local
            // and shared space, by looking at the DB and then checking what is
            // there
            // followed by a move.
            // Since we are doing a move, it should be ok to have this happend
            // on
            // the fly.

            try {
                migrateLocalSegments();
                migrateSharedSegments();
            } catch (IOException ex) {
                log.error(
                        "Failed to migrate search content to new format, the instance should not continue to run ",
                        ex);
                System.exit(-1);
            }

            /*
             * The storage is created by hibernate now try { if (autoDdl) {
             * SqlService.getInstance().ddl(this.getClass().getClassLoader(),
             * "search_cluster"); } } catch (Exception ex) { log.error("Failed
             * to init JDBCClusterIndexStorage", ex); }
             */
            log.info(this + ":init() Ok ");
        } catch (Exception ex) {
            log.error("Failed to start Cluster Index store", ex);
            System.exit(-1);
        }

    }

    /**
     * Ther might need to be some locking here. When readers use this, they will
     * update the local segments with more information, update the local
     * Segmetns from the DB
     * 
     * @param locked
     *        A locak has been taken on the index
     */
    public List<SegmentInfo> updateSegments() {
        Connection connection = null;
        List<SegmentInfo> segmentList = new ArrayList<SegmentInfo>();
        try {
            connection = dataSource.getConnection();
            List<SegmentInfo> dbSegments = getDBSegments(connection);
            if (log.isDebugEnabled())
                log.debug("Update: DB Segments = " + dbSegments.size());
            // remove files not in the dbSegmentList
            List<SegmentInfo> localSegments = getLocalSegments();

            List<SegmentInfo> badLocalSegments = getBadLocalSegments();
            // delete any bad local segments before we load so that they get
            // updated
            // from the db
            deleteAllSegments(badLocalSegments);

            List<SegmentInfo> deletedSegments = getDeletedLocalSegments();
            // delete any segments marked as for deletion, by the last cycle.
            // if this is due to a index reader event, we should not be
            // performing this operation
            // as the current reader will have these files open.
            // we should only delete the old segments update thread
            // If this call forms part of an exception, then we should look at
            // doing a
            // timeout on the delete files.
            deleteAllSegments(deletedSegments);

            if (log.isDebugEnabled())
                log.debug("Update: Local Segments = " + localSegments.size());

            // which of the dbSegments are not present locally
            List<SegmentInfo> updateLocalSegments = new ArrayList<SegmentInfo>();
            for (Iterator<SegmentInfo> i = dbSegments.iterator(); i.hasNext();) {
                SegmentInfo db_si = (SegmentInfo) i.next();
                boolean found = false;
                String name = db_si.getName();
                for (Iterator<SegmentInfo> j = localSegments.iterator(); j.hasNext();) {
                    SegmentInfo local_si = (SegmentInfo) j.next();
                    if (name.equals(local_si.getName())) {
                        found = true;
                        break;
                    }
                }
                if (!found) {
                    updateLocalSegments.add(db_si);
                    if (log.isDebugEnabled())
                        log.debug("Missing Will update " + db_si);
                } else {
                    if (log.isDebugEnabled())
                        log.debug("Present Will Not update " + db_si);
                }
            }

            // which of the dbsegmetnts are newer than local versions
            for (Iterator<SegmentInfo> i = localSegments.iterator(); i.hasNext();) {
                SegmentInfo current_si = (SegmentInfo) i.next();
                boolean found = false;
                String name = current_si.getName();
                long version = current_si.getVersion();
                for (Iterator<SegmentInfo> j = dbSegments.iterator(); j.hasNext();) {
                    SegmentInfo db_si = (SegmentInfo) j.next();
                    if (name.equals(db_si.getName()) && db_si.getVersion() > version) {
                        updateLocalSegments.add(db_si);
                        if (log.isDebugEnabled())
                            log.debug("Newer will Update " + db_si);
                        found = true;
                        break;
                    }
                }
                if (!found) {
                    if (log.isDebugEnabled())
                        log.debug("Ok will not update " + current_si);
                }
            }

            List<SegmentInfo> removeLocalSegments = new ArrayList<SegmentInfo>();

            // which segments exist locally but not in the DB, these should
            // be
            // removed
            for (Iterator<SegmentInfo> i = localSegments.iterator(); i.hasNext();) {

                SegmentInfo local_si = (SegmentInfo) i.next();
                // only check local segments that are not new and not
                if (local_si.isCreated()) {
                    boolean found = false;
                    String name = local_si.getName();
                    for (Iterator<SegmentInfo> j = dbSegments.iterator(); j.hasNext();) {
                        SegmentInfo db_si = (SegmentInfo) j.next();
                        if (name.equals(db_si.getName())) {
                            found = true;
                            break;
                        }
                    }
                    if (!found) {
                        removeLocalSegments.add(local_si);
                        if (log.isDebugEnabled())
                            log.debug("Will remove " + local_si);
                    } else {
                        if (log.isDebugEnabled())
                            log.debug("Ok Will not remove " + local_si);
                    }
                }
            }

            // if we could mark the local segment for deletion so that
            // its is only deleted the next time a lock is taken on the index
            for (Iterator<SegmentInfo> i = removeLocalSegments.iterator(); i.hasNext();) {
                SegmentInfo rmsi = (SegmentInfo) i.next();
                removeLocalSegment(rmsi);
            }

            // process the get list, first markign the segments as existing, so
            // that other threads
            // dont update them, then perform the update.
            try {
                for (Iterator<SegmentInfo> i = updateLocalSegments.iterator(); i.hasNext();) {
                    SegmentInfo addsi = (SegmentInfo) i.next();
                    addsi.lockLocalSegment();
                }
                for (Iterator<SegmentInfo> i = updateLocalSegments.iterator(); i.hasNext();) {
                    SegmentInfo addsi = (SegmentInfo) i.next();
                    try {
                        // This is thread safe strangely since the source doesnt
                        // change
                        // hence although it could be wastefull, more than one
                        // copy can perform an update at a
                        // time.
                        if (addsi.isLocalLock()) {
                            updateLocalSegment(connection, addsi);
                        } else {
                            log.warn("Not Updating Segment, since lock is not on this thread " + addsi.getName());
                        }
                    } catch (Exception ex) {
                        // ignore failures to unpack a local segment. It may
                        // have
                        // been removed by
                        // annother node
                        log.info("Segment was not unpacked " + ex.getClass().getName() + ":" + ex.getMessage());
                    }

                }
            } finally {
                for (Iterator<SegmentInfo> i = updateLocalSegments.iterator(); i.hasNext();) {
                    SegmentInfo addsi = (SegmentInfo) i.next();
                    addsi.unlockLocalSegment();
                }

            }
            // if we made any modifications, we also need to process the patch
            if (updateLocalSegments.size() > 0) {
                updateLocalPatch(connection);
            }

            // build the list putting the current segment at the end
            for (Iterator<SegmentInfo> i = dbSegments.iterator(); i.hasNext();) {
                SegmentInfo si = (SegmentInfo) i.next();
                File f = si.getSegmentLocation();
                if (f.exists()) {
                    // only add those segments that exist after the sync
                    segmentList.add(si);
                }
                if (log.isDebugEnabled())
                    log.debug("Segment Present at " + f.getName());
            }

            connection.commit();
        } catch (Exception sqle) {
            log.error("Failed to update segments ", sqle);
            try {
                connection.rollback();
            } catch (Exception ex) {
                log.debug(ex);
            }
        } finally {
            try {
                connection.close();
            } catch (Exception e) {
                log.debug(e);
            }
        }
        return segmentList;
    }

    private void deleteAllSegments(List<SegmentInfo> badLocalSegments) {
        for (Iterator<SegmentInfo> i = badLocalSegments.iterator(); i.hasNext();) {

            SegmentInfo s = i.next();
            s.doFinalDelete();
        }
    }

    /**
     * save the local segments to the DB
     */
    public List<SegmentInfo> saveSegments() {
        Connection connection = null;
        List<SegmentInfo> segmentList = new ArrayList<SegmentInfo>();
        try {
            connection = dataSource.getConnection();
            List<SegmentInfo> dbSegments = getDBSegments(connection);
            // remove files not in the dbSegmentList
            List<SegmentInfo> localSegments = getLocalSegments();
            List<SegmentInfo> badLocalSegments = getBadLocalSegments();

            // find the dbSegments that are not present locally

            List<SegmentInfo> removeDBSegments = new ArrayList<SegmentInfo>();
            List<SegmentInfo> currentDBSegments = new ArrayList<SegmentInfo>();

            // which segments exist inthe db but not locally
            for (Iterator<SegmentInfo> i = dbSegments.iterator(); i.hasNext();) {
                SegmentInfo db_si = (SegmentInfo) i.next();
                boolean found = false;
                String name = db_si.getName();
                for (Iterator<SegmentInfo> j = localSegments.iterator(); j.hasNext();) {
                    SegmentInfo local_si = (SegmentInfo) j.next();
                    if (name.equals(local_si.getName())) {
                        found = true;
                        break;
                    }
                }
                // dont delete bad segments from the DB
                if (!found) {
                    for (Iterator<SegmentInfo> j = badLocalSegments.iterator(); j.hasNext();) {
                        SegmentInfo local_file = (SegmentInfo) j.next();
                        if (name.equals(local_file.getName())) {
                            found = true;
                            break;
                        }
                    }
                }
                if (!found) {
                    removeDBSegments.add(db_si);
                    if (log.isDebugEnabled())
                        log.debug("Will remove from the DB " + db_si);
                } else {
                    currentDBSegments.add(db_si);
                    if (log.isDebugEnabled())
                        log.debug("In the DB will not remove " + db_si);
                }
            }

            List<SegmentInfo> updateDBSegments = new ArrayList<SegmentInfo>();
            // which of the localSegments are not in the db

            for (Iterator<SegmentInfo> i = localSegments.iterator(); i.hasNext();) {
                SegmentInfo local_si = (SegmentInfo) i.next();
                boolean found = false;
                String name = local_si.getName();
                for (Iterator<SegmentInfo> j = dbSegments.iterator(); j.hasNext();) {
                    SegmentInfo db_si = (SegmentInfo) j.next();
                    if (name.equals(db_si.getName())) {
                        found = true;
                        break;
                    }
                }
                if (!found) {
                    updateDBSegments.add(local_si);
                    if (log.isDebugEnabled())
                        log.debug(" Will update to the DB " + local_si);
                } else {
                    if (log.isDebugEnabled())
                        log.debug(" Will NOT update to the DB " + local_si);

                }
            }

            // which of the localSegments have been modified
            for (Iterator<SegmentInfo> i = localSegments.iterator(); i.hasNext();) {
                SegmentInfo local_si = (SegmentInfo) i.next();
                boolean found = false;
                String name = local_si.getName();
                long version = local_si.getVersion();
                for (Iterator<SegmentInfo> j = dbSegments.iterator(); j.hasNext();) {
                    SegmentInfo db_si = (SegmentInfo) j.next();
                    if (name.equals(db_si.getName()) && version > db_si.getVersion()) {
                        updateDBSegments.add(db_si);
                        if (log.isDebugEnabled())
                            log.debug("Will update modified to the DB " + db_si);
                        found = true;
                        break;
                    }
                }
                if (!found) {
                    if (log.isDebugEnabled())
                        log.debug("Will not update the DB, matches " + local_si);

                }
            }

            // process the get list
            for (Iterator<SegmentInfo> i = updateDBSegments.iterator(); i.hasNext();) {
                SegmentInfo addsi = (SegmentInfo) i.next();
                updateDBSegment(connection, addsi);
            }
            // build the list putting the current segment at the end
            updateDBPatch(connection);

            for (Iterator<SegmentInfo> i = updateDBSegments.iterator(); i.hasNext();) {
                SegmentInfo si = (SegmentInfo) i.next();
                File f = si.getSegmentLocation();
                segmentList.add(si);
                if (log.isDebugEnabled())
                    log.debug("Segments saved " + f.getName());

            }

            // process the remove list, the update was Ok so we can remove all
            // the old segments
            for (Iterator<SegmentInfo> i = removeDBSegments.iterator(); i.hasNext();) {
                SegmentInfo rmsi = (SegmentInfo) i.next();
                removeDBSegment(connection, rmsi);
            }
            connection.commit();
            deleteAllSegments(badLocalSegments);
        } catch (Exception ex) {
            log.error("Failed to Save Segments back to Central Storage", ex);
            try {
                connection.rollback();
            } catch (Exception e) {
                log.debug(e);
            }
            recoverFromFailure();
        } finally {
            try {
                connection.close();
            } catch (Exception e) {
                log.debug(e);
            }
        }
        return segmentList;
    }

    public List<SegmentInfo> saveAllSegments() {
        Connection connection = null;
        List<SegmentInfo> segmentList = new ArrayList<SegmentInfo>();
        try {
            connection = dataSource.getConnection();
            List<SegmentInfo> dbSegments = getDBSegments(connection);
            // remove files not in the dbSegmentList
            List<SegmentInfo> localSegments = getLocalSegments();
            List<SegmentInfo> badLocalSegments = getBadLocalSegments();

            // find the dbSegments that are not present locally

            List<SegmentInfo> updateDBSegments = new ArrayList<SegmentInfo>();
            // which of the localSegments are not in the db

            for (Iterator<SegmentInfo> i = localSegments.iterator(); i.hasNext();) {
                SegmentInfo local_si = (SegmentInfo) i.next();

                boolean found = false;
                String name = local_si.getName();
                for (Iterator<SegmentInfo> j = dbSegments.iterator(); j.hasNext();) {
                    SegmentInfo db_si = (SegmentInfo) j.next();
                    if (name.equals(db_si.getName())) {
                        found = true;
                        break;
                    }
                }
                // dont delete bad segments from the DB
                if (!found) {
                    for (Iterator<SegmentInfo> j = badLocalSegments.iterator(); j.hasNext();) {
                        SegmentInfo local_file = (SegmentInfo) j.next();
                        if (name.equals(local_file.getName())) {
                            found = true;
                            break;
                        }
                    }
                }

                if (!found) {
                    updateDBSegments.add(local_si);
                }
            }

            // the db segments
            for (Iterator<SegmentInfo> i = localSegments.iterator(); i.hasNext();) {
                SegmentInfo local_si = (SegmentInfo) i.next();
                String name = local_si.getName();

                for (Iterator<SegmentInfo> j = dbSegments.iterator(); j.hasNext();) {
                    SegmentInfo db_si = (SegmentInfo) j.next();
                    if (name.equals(db_si.getName())) {
                        updateDBSegments.add(db_si);
                        break;
                    }
                }
            }

            // process the get list
            for (Iterator<SegmentInfo> i = updateDBSegments.iterator(); i.hasNext();) {
                SegmentInfo addsi = (SegmentInfo) i.next();
                updateDBSegment(connection, addsi);
            }
            // build the list putting the current segment at the end

            for (Iterator<SegmentInfo> i = updateDBSegments.iterator(); i.hasNext();) {
                SegmentInfo si = (SegmentInfo) i.next();
                segmentList.add(si);
            }
            connection.commit();

            deleteAllSegments(badLocalSegments);
        } catch (Exception ex) {
            log.error("Failed to Save Segments back to Central Storage", ex);
            try {
                connection.rollback();
            } catch (Exception e) {
                log.debug(e);
            }
            recoverFromFailure();
        } finally {
            try {
                connection.close();
            } catch (Exception e) {
                log.debug(e);
            }
        }
        return segmentList;
    }

    protected void updateLocalSegment(Connection connection, SegmentInfo addsi) throws SQLException, IOException {
        if (searchService.hasDiagnostics()) {
            log.info("\tUpdate Local Segment from Database " + addsi);
        }
        if (localSegmentsOnly) {
            log.warn("Update Local Segment Requested with inactive Shared Storage " + addsi);
        } else {
            if (sharedSegments == null || sharedSegments.length() == 0) {
                updateLocalSegmentBLOB(connection, addsi);
            } else {
                updateLocalSegmentFilesystem(connection, addsi);
            }
        }

    }

    /**
     * updte a segment from the database
     * 
     * @param connection
     * @param addsi
     */
    protected void updateLocalSegmentBLOB(Connection connection, SegmentInfo addsi)
            throws SQLException, IOException {
        if (log.isDebugEnabled())
            log.debug("Updating local segment from databse " + addsi);
        PreparedStatement segmentSelect = null;
        ResultSet resultSet = null;
        try {
            segmentSelect = connection
                    .prepareStatement("select version_, packet_ from search_segments where name_ = ?");
            segmentSelect.clearParameters();
            segmentSelect.setString(1, addsi.getName());
            resultSet = segmentSelect.executeQuery();
            if (resultSet.next()) {
                InputStream packetStream = null;
                try {
                    long version = resultSet.getLong(1);
                    packetStream = resultSet.getBinaryStream(2);
                    clusterStorage.unpackSegment(addsi, packetStream, version);
                    if (log.isDebugEnabled())
                        log.debug("Updated Packet from DB to versiob " + version);
                } finally {
                    try {
                        packetStream.close();
                    } catch (Exception ex) {
                        log.debug(ex);
                    }
                }
            } else {
                log.error("Didnt find segment in database");
            }
        } finally {
            try {
                resultSet.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                segmentSelect.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
        }

    }

    /**
     * remove a local segment
     * 
     * @param rmsi
     */
    public void removeLocalSegment(SegmentInfo rmsi) {

        rmsi.setDeleted();
        if (searchService.hasDiagnostics()) {
            log.info("\tMarked Local Segment for deletion " + rmsi);
        }
    }

    /**
     * get a list of all DB segments ordered by version
     * 
     * @param connection
     * @return
     */
    private List<SegmentInfo> getDBSegments(Connection connection) throws SQLException {
        PreparedStatement segmentAllSelect = null;
        ResultSet resultSet = null;
        List<SegmentInfo> dbsegments = new ArrayList<SegmentInfo>();
        try {
            segmentAllSelect = connection
                    .prepareStatement("select version_, name_ from search_segments where name_ <> ? ");
            segmentAllSelect.clearParameters();
            segmentAllSelect.setString(1, INDEX_PATCHNAME);
            resultSet = segmentAllSelect.executeQuery();
            while (resultSet.next()) {
                final long version = resultSet.getLong(1);
                final String name = resultSet.getString(2);
                SegmentInfo si = SegmentInfoImpl.newSharedSegmentInfo(name, version, localStructuredStorage,
                        searchIndexDirectory);
                dbsegments.add(si);
                if (log.isDebugEnabled())
                    log.debug("DB Segment " + si);
            }
        } finally {
            try {
                resultSet.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                segmentAllSelect.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
        }
        return dbsegments;
    }

    protected void updateDBPatch(Connection connection) throws SQLException, IOException {

        if (localSegmentsOnly) {
            if (log.isDebugEnabled())
                log.debug("Update Patch Requested with inactive Shared Storage ");
        } else {
            if (sharedSegments == null || sharedSegments.length() == 0) {
                updateDBPatchBLOB(connection);
            } else {
                updateDBPatchFilesystem(connection);
            }
        }
    }

    /**
     * updat this save this local segment into the db
     * 
     * @param connection
     * @param addsi
     */
    protected void updateDBPatchBLOB(Connection connection) throws SQLException, IOException {

        PreparedStatement segmentUpdate = null;
        PreparedStatement segmentInsert = null;
        InputStream packetStream = null;
        File packetFile = null;
        long newVersion = System.currentTimeMillis();
        try {
            segmentUpdate = connection.prepareStatement(
                    "update search_segments set packet_ = ?, version_ = ?, size_ = ? where name_ = ?");
            segmentInsert = connection.prepareStatement(
                    "insert into search_segments (packet_, name_, version_, size_ ) values ( ?,?,?,?)");
            packetFile = clusterStorage.packPatch();
            if (packetFile.exists()) {
                packetStream = new FileInputStream(packetFile);
                segmentUpdate.clearParameters();
                segmentUpdate.setBinaryStream(1, packetStream, (int) packetFile.length());
                segmentUpdate.setLong(2, newVersion);
                segmentUpdate.setLong(3, packetFile.length());
                segmentUpdate.setString(4, INDEX_PATCHNAME);
                if (segmentUpdate.executeUpdate() != 1) {
                    segmentInsert.clearParameters();
                    segmentInsert.setBinaryStream(1, packetStream, (int) packetFile.length());
                    segmentInsert.setString(2, INDEX_PATCHNAME);
                    segmentInsert.setLong(3, newVersion);
                    segmentInsert.setLong(4, packetFile.length());
                    if (segmentInsert.executeUpdate() != 1) {
                        throw new SQLException(" Failed to insert patch  ");
                    }
                }
                if (log.isDebugEnabled())
                    log.debug("DB Updated Patch ");
            } else {
                log.warn(" Packed Patch does not exist " + packetFile.getPath());
            }
        } finally {
            try {
                if (packetStream != null) {
                    packetStream.close();
                }
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                packetFile.delete();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                segmentUpdate.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                segmentInsert.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
        }

    }

    /**
     * updat this save this local segment into the db
     * 
     * @param connection
     * @param addsi
     */
    protected void updateDBPatchFilesystem(Connection connection) throws SQLException, IOException {

        PreparedStatement segmentUpdate = null;
        PreparedStatement segmentInsert = null;
        FileChannel packetStream = null;
        FileInputStream packetFIS = null;
        FileChannel sharedStream = null;
        FileOutputStream sharedFOS = null;

        File packetFile = null;
        File sharedFinalFile = null;
        File sharedTempFile = null;
        long newVersion = System.currentTimeMillis();
        try {
            sharedTempFile = new File(getSharedTempFileName(INDEX_PATCHNAME));
            sharedFinalFile = new File(getSharedFileName(INDEX_PATCHNAME, sharedStructuredStorage));
            packetFile = clusterStorage.packPatch();
            if (packetFile.exists()) {
                packetFIS = new FileInputStream(packetFile);
                packetStream = packetFIS.getChannel();
                File sharedTempFileParent = sharedTempFile.getParentFile();
                if (!sharedTempFileParent.exists() && !sharedTempFileParent.mkdirs()) {
                    log.warn("couldn't create " + sharedTempFileParent.getPath());
                }
                sharedFOS = new FileOutputStream(sharedTempFile);
                sharedStream = sharedFOS.getChannel();

                doBlockedStream(packetStream, sharedStream);

                packetStream.close();
                sharedStream.close();

                segmentUpdate = connection
                        .prepareStatement("update search_segments set  version_ = ?, size_ = ? where name_ = ? ");
                segmentInsert = connection
                        .prepareStatement("insert into search_segments ( name_, version_, size_ ) values ( ?,?,?)");

                segmentUpdate.clearParameters();
                segmentUpdate.setLong(1, newVersion);
                segmentUpdate.setLong(2, packetFile.length());
                segmentUpdate.setString(3, INDEX_PATCHNAME);
                if (segmentUpdate.executeUpdate() != 1) {
                    segmentInsert.clearParameters();
                    segmentInsert.setString(1, INDEX_PATCHNAME);
                    segmentInsert.setLong(2, newVersion);
                    segmentInsert.setLong(3, packetFile.length());
                    if (segmentInsert.executeUpdate() != 1) {
                        throw new SQLException(" Failed to add patch packet  ");
                    }
                }

                long st = System.currentTimeMillis();
                if (!sharedTempFile.renameTo(sharedFinalFile)) {
                    log.warn("Couldn't rename file " + sharedTempFile.getPath() + " to "
                            + sharedFinalFile.getPath());
                }
                if (searchService.hasDiagnostics()) {
                    log.info("Renamed " + sharedTempFile.getPath() + " to " + sharedFinalFile.getPath() + " in "
                            + (System.currentTimeMillis() - st) + "ms");
                }
            } else {
                log.warn("Packet file does not exist " + packetFile.getPath());
            }

        } finally {

            try {
                if (packetStream != null) {
                    packetStream.close();
                    packetFIS.close();
                }
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                packetFile.delete();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                if (sharedStream != null) {
                    sharedStream.close();
                    sharedFOS.close();
                }
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                sharedTempFile.delete();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                segmentUpdate.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                segmentInsert.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
        }

    }

    protected void updateDBSegment(Connection connection, SegmentInfo addsi) throws SQLException, IOException {
        if (searchService.hasDiagnostics()) {
            log.info("\tUpdate Database Segment from Local " + addsi);
        }
        if (localSegmentsOnly) {
            if (log.isDebugEnabled())
                log.debug("Not Saving Segment to DB as no Shared Storage " + addsi);
        } else {
            if (sharedSegments == null || sharedSegments.length() == 0) {
                updateDBSegmentBLOB(connection, addsi);
            } else {
                updateDBSegmentFilesystem(connection, addsi);
            }
        }
    }

    /**
     * updat this save this local segment into the db
     * 
     * @param connection
     * @param addsi
     */
    protected void updateDBSegmentBLOB(Connection connection, SegmentInfo addsi) throws SQLException, IOException {

        PreparedStatement segmentUpdate = null;
        PreparedStatement segmentInsert = null;
        InputStream packetStream = null;
        File packetFile = null;
        long newVersion = System.currentTimeMillis();
        try {
            segmentUpdate = connection.prepareStatement(
                    "update search_segments set packet_ = ?, version_ = ?, size_ = ? where name_ = ? and version_ = ?");
            segmentInsert = connection.prepareStatement(
                    "insert into search_segments (packet_, name_, version_, size_ ) values ( ?,?,?,?)");
            packetFile = clusterStorage.packSegment(addsi, newVersion);
            if (packetFile.exists()) {
                packetStream = new FileInputStream(packetFile);
                if (addsi.isInDb()) {
                    segmentUpdate.clearParameters();
                    segmentUpdate.setBinaryStream(1, packetStream, (int) packetFile.length());
                    segmentUpdate.setLong(2, newVersion);
                    segmentUpdate.setLong(3, packetFile.length());
                    segmentUpdate.setString(4, addsi.getName());
                    segmentUpdate.setLong(5, addsi.getVersion());
                    if (segmentUpdate.executeUpdate() != 1) {
                        throw new SQLException(" ant Find packet to update " + addsi);
                    }
                } else {
                    segmentInsert.clearParameters();
                    segmentInsert.setBinaryStream(1, packetStream, (int) packetFile.length());
                    segmentInsert.setString(2, addsi.getName());
                    segmentInsert.setLong(3, newVersion);
                    segmentInsert.setLong(4, packetFile.length());
                    if (segmentInsert.executeUpdate() != 1) {
                        throw new SQLException(" Failed to insert packet  " + addsi);
                    }
                }
                addsi.setVersion(newVersion);
                if (log.isDebugEnabled())
                    log.debug("DB Updated " + addsi);
                try {
                    packetStream.close();
                } catch (Exception ex) {
                    log.debug(ex);
                }
                try {
                    packetFile.delete();
                } catch (Exception ex) {
                    log.debug(ex);
                }
            } else {
                log.warn("Packet file does not exist " + packetFile.getPath());
            }

        } finally {
            try {
                packetStream.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                packetFile.delete();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                segmentUpdate.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                segmentInsert.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
        }

    }

    private void removeDBSegment(Connection connection, SegmentInfo rmsi) throws SQLException {
        PreparedStatement segmentDelete = null;
        try {
            if (rmsi.isInDb()) {
                segmentDelete = connection
                        .prepareStatement("delete from search_segments where name_ = ? and version_ = ?");
                segmentDelete.clearParameters();
                segmentDelete.setString(1, rmsi.getName());
                segmentDelete.setLong(2, rmsi.getVersion());
                segmentDelete.execute();

                String sharedSegment = getSharedFileName(rmsi.getName(), sharedStructuredStorage);
                if (sharedSegment != null) {
                    File f = new File(sharedSegment);
                    if (f.exists()) {
                        if (!f.delete()) {
                            log.warn("unable to delete " + f.getPath());
                        }
                    }
                }

                if (searchService.hasDiagnostics()) {
                    log.info("\tRemoved Segment From Database [" + rmsi + "]");
                }
            }
        } finally {
            try {
                if (segmentDelete != null) {
                    segmentDelete.close();
                }
            } catch (Exception ex) {
                log.debug(ex);
            }
        }
    }

    /**
     * create a new local segment and mark its tiestamp
     */
    public SegmentInfo newSegment() throws IOException {
        File f = null;
        for (;;) {
            f = SegmentInfoImpl.getSegmentLocation(String.valueOf(System.currentTimeMillis()),
                    localStructuredStorage, searchIndexDirectory);
            if (!f.exists()) {
                break;
            }
        }
        if (!f.mkdirs()) {
            log.warn("unable to create directory: " + f.getPath());
        }

        SegmentInfo si = SegmentInfoImpl.newLocalSegmentInfo(f, localStructuredStorage, searchIndexDirectory);
        si.setNew();
        si.setTimeStamp(System.currentTimeMillis());

        return si;
    }

    /**
     * get a list of local segments
     * 
     * @return
     * @throws IOException
     */
    public List<SegmentInfo> getLocalSegments() throws IOException {
        List<SegmentInfo> l = new ArrayList<SegmentInfo>();
        File searchDir = new File(searchIndexDirectory);
        return getLocalSegments(searchDir, l);
    }

    /**
     * recurse into a list of segments
     * 
     * @param searchDir
     * @param l
     * @return
     * @throws IOException
     */
    public List<SegmentInfo> getLocalSegments(File searchDir, List<SegmentInfo> l) throws IOException {

        File[] files = searchDir.listFiles();
        if (files != null) {
            for (int i = 0; i < files.length; i++) {
                if (files[i].isDirectory()) {

                    SegmentInfo sgi = SegmentInfoImpl.newLocalSegmentInfo(files[i], localStructuredStorage,
                            searchIndexDirectory);
                    if (sgi.isClusterSegment()) {
                        if (IndexReader.indexExists(FSDirectory.open(files[i]))) {

                            if (sgi.isCreated()) {
                                l.add(sgi);
                                if (log.isDebugEnabled())
                                    log.debug("LO Segment " + sgi);
                            } else {
                                if (log.isDebugEnabled())
                                    log.debug("LO Segment not created " + sgi.toString());
                            }
                        } else {
                            log.warn("Found Orphaned directory with no segment information present " + files[i]);
                        }

                    } else {
                        l = getLocalSegments(files[i], l);
                    }
                }
            }
        }
        return l;
    }

    /**
     * get a list of bad segmetns with brokenindexes
     * 
     * @return
     * @throws IOException
     */
    private List<SegmentInfo> getBadLocalSegments() throws IOException {
        List<SegmentInfo> l = new ArrayList<SegmentInfo>();
        File searchDir = new File(searchIndexDirectory);
        return getBadLocalSegments(searchDir, l);
    }

    /**
     * get a list of segments that are ready for deletion
     * 
     * @return
     * @throws IOException
     */
    private List<SegmentInfo> getDeletedLocalSegments() throws IOException {
        List<SegmentInfo> l = new ArrayList<SegmentInfo>();
        File searchDir = new File(searchIndexDirectory);
        return getDeletedLocalSegments(searchDir, l);
    }

    /**
     * recurse into a list of bad local segments
     * 
     * @param searchDir
     * @param l
     * @return
     * @throws IOException
     */
    private List<SegmentInfo> getBadLocalSegments(File searchDir, List<SegmentInfo> l) throws IOException {
        if (searchDir.isDirectory()) {
            File[] files = searchDir.listFiles();
            if (files != null) {
                for (int i = 0; i < files.length; i++) {
                    SegmentInfo sgi = SegmentInfoImpl.newLocalSegmentInfo(files[i], localStructuredStorage,
                            searchIndexDirectory);
                    if (sgi.isClusterSegment()) {
                        if (sgi.isCreated()) {
                            if (!IndexReader.indexExists(FSDirectory.open(files[i]))) {
                                l.add(sgi);
                            }

                        }
                    } else {
                        l = getBadLocalSegments(files[i], l);
                    }
                }
            }
        }

        return l;
    }

    /**
     * Get a list of segments to be deleted
     * 
     * @param searchDir
     * @param l
     * @return
     * @throws IOException
     */
    private List<SegmentInfo> getDeletedLocalSegments(File searchDir, List<SegmentInfo> l) throws IOException {
        if (searchDir.isDirectory()) {
            File[] files = searchDir.listFiles();
            if (files != null) {
                for (int i = 0; i < files.length; i++) {
                    SegmentInfo sgi = SegmentInfoImpl.newLocalSegmentInfo(files[i], localStructuredStorage,
                            searchIndexDirectory);
                    if (sgi.isClusterSegment()) {
                        if (sgi.isDeleted()) {
                            l.add(sgi);
                        }
                    } else {
                        l = getDeletedLocalSegments(files[i], l);
                    }
                }
            }
        }
        return l;
    }

    /**
     * recover from a failiure
     */
    private void recoverFromFailure() {
        log.error("Recover from Failiure is not implementated at the moment,"
                + " the local index is corrupt, please delete it and it will " + "reload from the database");

    }

    /**
     * @return Returns the dataSource.
     */
    public DataSource getDataSource() {
        return dataSource;
    }

    /**
     * @param dataSource
     *        The dataSource to set.
     */
    public void setDataSource(DataSource dataSource) {
        this.dataSource = dataSource;
    }

    public void setLocation(String location) {

        searchIndexDirectory = location;
        log.info("Search Index Location is " + location);

    }

    /**
     * @param autoDdl
     *        The autoDdl to set.
     */
    public void setAutoDdl(boolean autoDdl) {
        this.autoDdl = autoDdl;
    }

    /**
     * create a temporary index for indexing operations
     */
    public File getTemporarySegment(boolean delete) {
        // this index will not have a timestamp, and hence will not be part sync
        // with the db
        File f = new File(searchIndexDirectory, TEMP_INDEX_NAME);
        if (delete && f.exists()) {
            SegmentInfoImpl.deleteAll(f);
        }
        if (!f.exists() && !f.mkdirs()) {
            log.warn("couldn't create directory " + f.getPath());
        }
        return f;
    }

    public void removeTemporarySegment() {
        File f = new File(searchIndexDirectory, TEMP_INDEX_NAME);
        if (f.exists()) {
            SegmentInfoImpl.deleteAll(f);
        }
    }

    public SegmentInfo saveTemporarySegment() throws IOException {
        SegmentInfo segInfo = newSegment();
        File s = new File(searchIndexDirectory, TEMP_INDEX_NAME);
        File d = segInfo.getSegmentLocation();
        copyAll(s, d);
        segInfo.setCreated();
        segInfo.touchSegment();
        return segInfo;
    }

    /**
     * Copy a file to a directory, if the source is a directory, the copy
     * recurses into the directory. If the destination does not exists, it is
     * assumed to be a file.
     * 
     * @param s
     *        the source file
     * @param d
     *        the source directory
     * @throws IOException
     */
    private void copyAll(File s, File d) throws IOException {
        if (s.isDirectory()) {
            File[] fl = s.listFiles();
            for (int i = 0; i < fl.length; i++) {
                if (fl[i].isFile()) {
                    copyFile(fl[i], d);
                } else {
                    File nd = new File(d, fl[i].getName());
                    if (!nd.mkdirs()) {
                        log.warn("couldn't create directories " + nd.getPath());
                    }
                    copyAll(fl[i], nd);
                }
            }
        } else {
            copyFile(s, d);
        }
    }

    /**
     * Copy a file from s to d, s will be a file, d may be a file or directory
     * 
     * @param s
     * @param d
     * @throws IOException
     */
    private void copyFile(File s, File d) throws IOException {
        if (log.isDebugEnabled())
            log.debug("Copying " + s.getAbsolutePath() + " to " + d.getAbsolutePath());
        if (s.exists() && s.isFile()) {
            File t = d; // target
            if (d.isDirectory()) {
                if (!d.exists()) {
                    if (!d.mkdirs()) {
                        log.warn("Unable to create directory " + d.getPath());
                    }
                }
                t = new File(d, s.getName());
            } else {
                File p = d.getParentFile();
                if (!p.exists()) {
                    if (!p.mkdirs()) {
                        log.warn("couldn't create:  " + p.getPath());
                    }
                }
            }
            FileChannel srcChannel = null;
            FileChannel dstChannel = null;
            try {
                // use nio
                // Create channel on the source
                srcChannel = new FileInputStream(s).getChannel();

                // Create channel on the destination
                dstChannel = new FileOutputStream(t).getChannel();

                // Copy file contents from source to destination
                doBlockedStream(srcChannel, dstChannel);

                // Close the channels
            } finally {
                try {
                    srcChannel.close();
                } catch (Exception ex) {
                    log.debug(ex);

                }
                try {
                    dstChannel.close();
                } catch (Exception ex) {
                    log.debug(ex);

                }
            }

        }
    }

    public void recoverSegment(SegmentInfo recoverSegInfo) {
        SegmentInfo segInfo = SegmentInfoImpl.newLocalSegmentInfo(recoverSegInfo);
        segInfo.debugSegment("Pre Recovery Check :");

        segInfo.compareTo("Comparing Disk Segment to Recovered Segment", recoverSegInfo);

        recoverSegInfo.setDeleted();
        recoverSegInfo.doFinalDelete();
        recoverSegInfo.setNew();
        Connection connection = null;
        try {
            connection = dataSource.getConnection();
            updateLocalSegment(connection, recoverSegInfo);
            // we also need to re-apply the patch
            updateLocalPatch(connection);
            connection.commit();
        } catch (Exception ex) {
            try {
                if (connection != null) {
                    connection.rollback();
                }
            } catch (Exception e) {
                log.debug(ex);
            }
            throw new RuntimeException("Failed to recover dammaged segment ", ex);

        } finally {
            try {
                if (connection != null) {
                    connection.close();
                }
            } catch (Exception e) {
                log.debug(e);

            }
        }

        SegmentInfo newSegInfo = SegmentInfoImpl.newLocalSegmentInfo(recoverSegInfo);
        newSegInfo.debugSegment("Recovered Segment");
        newSegInfo.compareTo("Comparing Recoverd Segment to Previous Disk Segment", segInfo);

    }

    protected void updateLocalPatch(Connection connection) throws SQLException, IOException {
        if (localSegmentsOnly) {
            log.warn("Update Patch Requested with inactive Shared Storage ");
        } else {
            if (sharedSegments == null || sharedSegments.length() == 0) {
                updateLocalPatchBLOB(connection);
            } else {
                updateLocalPatchFilesystem(connection);
            }
        }
    }

    protected void updateLocalPatchFilesystem(Connection connection) throws SQLException, IOException {
        if (log.isDebugEnabled())
            log.debug("Updating local patch ");
        PreparedStatement segmentSelect = null;
        ResultSet resultSet = null;
        try {
            segmentSelect = connection.prepareStatement("select version_ from search_segments where name_ = ?");
            segmentSelect.clearParameters();
            segmentSelect.setString(1, INDEX_PATCHNAME);
            resultSet = segmentSelect.executeQuery();
            if (resultSet.next()) {
                InputStream packetStream = null;
                try {

                    File f = new File(getSharedFileName(INDEX_PATCHNAME, sharedStructuredStorage));
                    if (f.exists()) {
                        packetStream = new FileInputStream(f);
                        clusterStorage.unpackPatch(packetStream);
                        if (log.isDebugEnabled())
                            log.debug("Updated Patch ");
                    } else {
                        log.warn("Shared Segment File does not exist " + f.getPath());
                    }
                } finally {
                    try {
                        if (packetStream != null) {
                            packetStream.close();
                        }
                    } catch (Exception ex) {
                        log.debug(ex);
                    }
                }
            } else {
                if (log.isDebugEnabled())
                    log.debug("Didnt find patch in database, this is Ok");
            }
        } finally {
            try {
                resultSet.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                segmentSelect.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
        }

    }

    private void updateLocalPatchBLOB(Connection connection) throws SQLException, IOException {
        if (log.isDebugEnabled())
            log.debug("Updating local patch ");
        PreparedStatement segmentSelect = null;
        ResultSet resultSet = null;
        try {
            segmentSelect = connection
                    .prepareStatement("select version_, packet_ from search_segments where name_ = ?");
            segmentSelect.clearParameters();
            segmentSelect.setString(1, INDEX_PATCHNAME);
            resultSet = segmentSelect.executeQuery();
            if (resultSet.next()) {
                InputStream packetStream = null;
                try {
                    long version = resultSet.getLong(1);
                    packetStream = resultSet.getBinaryStream(2);
                    clusterStorage.unpackPatch(packetStream);
                    if (log.isDebugEnabled())
                        log.debug("Updated Patch from DB " + version);
                } finally {
                    try {
                        packetStream.close();
                    } catch (Exception ex) {
                        log.debug(ex);
                    }
                }
            } else {
                if (log.isDebugEnabled())
                    log.debug("Didnt find patch in database, this is Ok ");
            }
        } finally {
            try {
                resultSet.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                segmentSelect.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
        }

    }

    public String getSegmentName(String segmentPath) {
        File f = new File(segmentPath);
        return f.getName();
    }

    /**
     * @return Returns the validate.
     */
    public boolean isValidate() {
        return validate;
    }

    /**
     * @param validate
     *        The validate to set.
     */
    public void setValidate(boolean validate) {
        this.validate = validate;
    }

    /**
     * updat this save this local segment into the db
     * 
     * @param connection
     * @param addsi
     */
    protected void updateDBSegmentFilesystem(Connection connection, SegmentInfo addsi)
            throws SQLException, IOException {

        PreparedStatement segmentUpdate = null;
        PreparedStatement segmentInsert = null;
        FileChannel packetStream = null;
        FileInputStream packetFIS = null;
        FileChannel sharedStream = null;
        FileOutputStream sharedFOS = null;
        File packetFile = null;
        File sharedFinalFile = null;
        File sharedTempFile = null;
        long newVersion = System.currentTimeMillis();
        try {
            sharedTempFile = new File(getSharedTempFileName(addsi.getName()));
            sharedFinalFile = new File(getSharedFileName(addsi.getName(), sharedStructuredStorage));
            packetFile = clusterStorage.packSegment(addsi, newVersion);
            if (packetFile.exists()) {
                packetFIS = new FileInputStream(packetFile);
                packetStream = packetFIS.getChannel();
                File parentFile = sharedTempFile.getParentFile();
                if (!parentFile.exists() && !parentFile.mkdirs()) {
                    log.warn("Unable to create directory " + sharedTempFile.getParentFile().getPath());
                }
                sharedFOS = new FileOutputStream(sharedTempFile);
                sharedStream = sharedFOS.getChannel();

                // Copy file contents from source to destination
                doBlockedStream(packetStream, sharedStream);

                packetStream.close();
                sharedStream.close();

                segmentUpdate = connection.prepareStatement(
                        "update search_segments set  version_ = ?, size_ = ? where name_ = ? and version_ = ?");
                segmentInsert = connection
                        .prepareStatement("insert into search_segments ( name_, version_, size_ ) values ( ?,?,?)");
                if (addsi.isInDb()) {
                    segmentUpdate.clearParameters();
                    segmentUpdate.setLong(1, newVersion);
                    segmentUpdate.setLong(2, packetFile.length());
                    segmentUpdate.setString(3, addsi.getName());
                    segmentUpdate.setLong(4, addsi.getVersion());
                    if (segmentUpdate.executeUpdate() != 1) {
                        throw new SQLException(" ant Find packet to update " + addsi);
                    }
                } else {
                    segmentInsert.clearParameters();
                    segmentInsert.setString(1, addsi.getName());
                    segmentInsert.setLong(2, newVersion);
                    segmentInsert.setLong(3, packetFile.length());
                    if (segmentInsert.executeUpdate() != 1) {
                        throw new SQLException(" Failed to insert packet  " + addsi);
                    }
                }
                addsi.setVersion(newVersion);
                File sharedParentFile = sharedFinalFile.getParentFile();
                if (!sharedParentFile.exists() && !sharedParentFile.mkdirs()) {
                    log.warn("Couln't create directory " + sharedParentFile.getPath());
                }
                long st = System.currentTimeMillis();
                if (!sharedTempFile.renameTo(sharedFinalFile)) {
                    log.warn("Couldn't rename " + sharedTempFile.getPath() + " to " + sharedFinalFile.getPath());

                }
                if (searchService.hasDiagnostics()) {
                    log.info("Renamed " + sharedTempFile.getPath() + " to " + sharedFinalFile.getPath() + " in "
                            + (System.currentTimeMillis() - st) + "ms");
                }

                log.info("DB Updated " + addsi);
            } else {
                log.warn("Packet file does not exist " + packetFile.getPath());
            }

        } finally {
            try {
                packetStream.close();
                packetFIS.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                packetFile.delete();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                sharedStream.close();
                sharedFOS.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                sharedTempFile.delete();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                segmentUpdate.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                segmentInsert.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
        }

    }

    /**
     * @param packetStream
     * @param sharedStream
     * @throws IOException
     */
    private void doBlockedStream(FileChannel from, FileChannel to) throws IOException {
        to.position(0);
        long size = from.size();
        for (long pos = 0; pos < size;) {
            long count = size - pos;
            if (count > MAX_BLOCK_SIZE) {
                count = MAX_BLOCK_SIZE;

            }

            to.position(pos);
            long cpos = to.position();
            log.debug("NIOTransfering |" + count + "| bytes from |" + pos + "| to |" + cpos + "|");
            long t = to.transferFrom(from, pos, count);
            pos = pos + t;
        }
        log.debug("  Final Size Source        " + from.size());
        log.debug("  Final Size Destination   " + to.size());
    }

    private String getSharedFileName(String name, boolean structured) {
        if (localSegmentsOnly) {
            return null;
        }
        if (sharedSegments != null && sharedSegments.length() > 0) {
            if (!sharedSegments.endsWith("/")) {
                sharedSegments = sharedSegments + "/";
            }
            if (structured && !INDEX_PATCHNAME.equals(name)) {
                String hashName = name.substring(name.length() - 4, name.length() - 2);
                return sharedSegments + hashName + "/" + name + ".zip";
            } else {
                return sharedSegments + name + ".zip";
            }
        }
        return null;
    }

    private String getSharedTempFileName(String name) {

        if (sharedSegments != null && sharedSegments.length() > 0) {
            if (!sharedSegments.endsWith("/")) {
                sharedSegments = sharedSegments + "/";
            }
            return sharedSegments + name + ".zip." + System.currentTimeMillis();
        }
        return null;
    }

    /**
     * updte a segment from the database
     * 
     * @param connection
     * @param addsi
     */
    protected void updateLocalSegmentFilesystem(Connection connection, SegmentInfo addsi)
            throws SQLException, IOException {
        if (log.isDebugEnabled())
            log.debug("Updating local segment from databse " + addsi);
        PreparedStatement segmentSelect = null;
        ResultSet resultSet = null;
        try {
            segmentSelect = connection.prepareStatement("select version_ from search_segments where name_ = ?");
            segmentSelect.clearParameters();
            segmentSelect.setString(1, addsi.getName());
            resultSet = segmentSelect.executeQuery();
            if (resultSet.next()) {
                InputStream packetStream = null;
                try {
                    long version = resultSet.getLong(1);
                    File f = new File(getSharedFileName(addsi.getName(), sharedStructuredStorage));
                    if (f.exists()) {
                        packetStream = new FileInputStream(f);
                        clusterStorage.unpackSegment(addsi, packetStream, version);
                        if (log.isDebugEnabled())
                            log.debug("Updated Local " + addsi);
                    } else {
                        log.warn("Shared Segment file is missing " + f.getPath());
                    }
                } finally {
                    try {
                        packetStream.close();
                    } catch (Exception ex) {
                        log.debug(ex);
                    }
                }
            } else {
                log.error("Didnt find segment in database");
            }
        } finally {
            try {
                resultSet.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                segmentSelect.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
        }

    }

    public String getSharedSegments() {
        return sharedSegments;
    }

    public void setSharedSegments(String sharedSegments) {
        this.sharedSegments = sharedSegments;
    }

    public void dolog(String message) {
        if (debug) {
            log.info("JDBCClusterDebug :" + message);
        } else if (log.isDebugEnabled()) {
            log.debug("JDBCClusterDebug :" + message);
        }
    }

    public long getLastUpdate() {
        PreparedStatement segmentSelect = null;
        ResultSet resultSet = null;
        Connection connection = null;

        try {
            connection = dataSource.getConnection();
            segmentSelect = connection
                    .prepareStatement("select version_ from search_segments order by version_ desc");
            segmentSelect.clearParameters();
            resultSet = segmentSelect.executeQuery();
            if (resultSet.next()) {
                return resultSet.getLong(1);
            } else {
                return 0;
            }
        } catch (Exception ex) {
            log.warn(" Cant find last update time " + ex.getClass().getName() + ":" + ex.getMessage());
            return 0;
        } finally {
            try {
                resultSet.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                segmentSelect.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
            try {
                connection.close();
            } catch (Exception ex) {
                log.debug(ex);

            }
        }
    }

    public List<Object[]> getSegmentInfoList() {
        List<Object[]> seginfo = new ArrayList<Object[]>();
        try {

            File searchDir = new File(searchIndexDirectory);
            long tsize = getSegmentInfoList(searchDir, seginfo);
            String size = null;
            if (tsize > 1024 * 1024 * 10) {
                size = String.valueOf(tsize / (1024 * 1024)) + "MB";
            } else if (tsize >= 1024 * 1024) {
                size = String.valueOf(tsize / (1024 * 1024)) + "." + String.valueOf(tsize / (102 * 1024) + "MB");
            } else {
                size = String.valueOf(tsize / (1024)) + "KB";
            }
            seginfo.add(new Object[] { "Total", size, "" });

        } catch (Exception ex) {
            seginfo.add(new Object[] {
                    "Failed to get Segment Info list " + ex.getClass().getName() + " " + ex.getMessage() });
        }
        return seginfo;

    }

    public long getSegmentInfoList(File searchDir, List<Object[]> seginfo) {

        File[] files = searchDir.listFiles();
        long tsize = 0;
        if (files != null) {
            for (int i = 0; i < files.length; i++) {
                if (files[i].isDirectory()) {

                    SegmentInfo sgi = SegmentInfoImpl.newLocalSegmentInfo(files[i], localStructuredStorage,
                            searchIndexDirectory);
                    if (sgi != null && sgi.isClusterSegment()) {
                        String name = files[i].getName();
                        long lsize = sgi.getLocalSegmentSize();
                        tsize += lsize;
                        long ts = sgi.getLocalSegmentLastModified();
                        String lastup = (new Date(ts)).toString();

                        String size = null;
                        if (lsize > 1024 * 1024 * 10) {
                            size = String.valueOf(lsize / (1024 * 1024)) + "MB";
                        } else if (lsize >= 1024 * 1024) {
                            size = String.valueOf(lsize / (1024 * 1024)) + "."
                                    + String.valueOf(lsize / (102 * 1024) + "MB");
                        } else {
                            size = String.valueOf(lsize / (1024)) + "KB";
                        }
                        seginfo.add(new Object[] { name, size, lastup });
                    } else {
                        tsize += getSegmentInfoList(files[i], seginfo);
                    }
                }
            }
        }
        return tsize;

    }

    private void migrateSharedSegments() {
        if (localSegmentsOnly) {
            return;
        }
        if (sharedSegments != null && sharedSegments.length() > 0) {
            Connection connection = null;
            try {
                connection = dataSource.getConnection();

                List<SegmentInfo> l = getDBSegments(connection);
                for (Iterator<SegmentInfo> li = l.iterator(); li.hasNext();) {
                    SegmentInfo si = (SegmentInfo) li.next();
                    String shared = getSharedFileName(si.getName(), !sharedStructuredStorage);
                    File f = new File(shared);
                    if (f.exists()) {
                        File fnew = new File(getSharedFileName(si.getName(), sharedStructuredStorage));
                        if (!fnew.getParentFile().mkdirs()) {
                            log.warn("couldn't create directory: " + fnew.getParentFile().getPath());
                        }
                        log.info("Moving " + f.getPath() + " to " + fnew.getPath());
                        if (!f.renameTo(fnew)) {
                            log.warn("Failed rename " + f.getPath() + " to " + fnew.getPath());
                        }
                    }
                }
                connection.commit();
            } catch (Exception ex) {
                try {
                    connection.rollback();
                } catch (Exception ex1) {
                    log.debug(ex);

                }

            } finally {
                try {
                    connection.close();
                } catch (Exception ex) {
                    log.debug(ex);
                }
            }
        }

    }

    private void migrateLocalSegments() throws IOException {
        List<SegmentInfo> l = getLocalSegments();
        for (Iterator<SegmentInfo> li = l.iterator(); li.hasNext();) {
            SegmentInfo si = li.next();
            File f = SegmentInfoImpl.getSegmentLocation(si.getName(), !localStructuredStorage,
                    searchIndexDirectory);
            if (f.exists()) {
                File fnew = SegmentInfoImpl.getSegmentLocation(si.getName(), localStructuredStorage,
                        searchIndexDirectory);
                if (!fnew.getParentFile().mkdirs()) {
                    log.warn("couldn't create directory " + fnew.getParentFile().getPath());
                }
                log.info("Moving " + f.getPath() + " to " + fnew.getPath());
                if (!f.renameTo(fnew)) {
                    log.warn("Failed to rename " + f.getPath() + " to " + fnew.getPath());
                }
            }
        }
    }

    public void getLock() throws IOException {

        if (parallelIndex) {
            throw new RuntimeException("Parallel index is not implemented yet");
        }

    }

    public void releaseLock() {
        if (parallelIndex) {
            throw new RuntimeException("Parallel index is not implemented yet");
        }
    }

    public boolean isMultipleIndexers() {
        return parallelIndex;
    }

    public boolean isParallelIndex() {
        return parallelIndex;
    }

    public void setParallelIndex(boolean parallelIndex) {
        this.parallelIndex = parallelIndex;
    }

    /**
     * @return Returns the localStructuredStorage.
     */
    public boolean isLocalStructuredStorage() {
        return localStructuredStorage;
    }

    /**
     * @param localStructuredStorage
     *        The localStructuredStorage to set.
     */
    public void setLocalStructuredStorage(boolean localStructuredStorage) {
        this.localStructuredStorage = localStructuredStorage;
    }

    /**
     * @return Returns the sharedStructuredStorage.
     */
    public boolean isSharedStructuredStorage() {
        return sharedStructuredStorage;
    }

    /**
     * @param sharedStructuredStorage
     *        The sharedStructuredStorage to set.
     */
    public void setSharedStructuredStorage(boolean sharedStructuredStorage) {
        this.sharedStructuredStorage = sharedStructuredStorage;
    }

    /**
     * @return the localSegmentsOnly
     */
    public boolean isLocalSegmentsOnly() {
        return localSegmentsOnly;
    }

    /**
     * @param localSegmentsOnly
     *        the localSegmentsOnly to set
     */
    public void setLocalSegmentsOnly(boolean localSegmentsOnly) {
        this.localSegmentsOnly = localSegmentsOnly;
    }

    /**
     * @return the searchService
     */
    public SearchService getSearchService() {
        return searchService;
    }

    /**
     * @param searchService
     *        the searchService to set
     */
    public void setSearchService(SearchService searchService) {
        this.searchService = searchService;
    }

    /*
     * (non-Javadoc)
     * 
     * @see org.sakaiproject.search.index.ClusterFilesystem#centralIndexExists()
     */
    public boolean centralIndexExists() {
        Connection connection = null;
        try {
            connection = dataSource.getConnection();
            List<SegmentInfo> l = getDBSegments(connection);
            if (l != null && l.size() > 0) {
                return true;
            }
            return false;
        } catch (SQLException e) {
            return false;
        } finally {
            try {
                connection.close();
            } catch (Exception ex) {
                log.debug(ex);
            }
        }
    }

}