Java tutorial
/********************************************************************************** * $URL: https://source.sakaiproject.org/svn/search/trunk/search-impl/impl/src/java/org/sakaiproject/search/index/impl/JDBCClusterIndexStore.java $ * $Id: JDBCClusterIndexStore.java 111640 2012-08-20 12:58:11Z david.horwitz@uct.ac.za $ *********************************************************************************** * * Copyright (c) 2003, 2004, 2005, 2006, 2007, 2008 The Sakai Foundation * * Licensed under the Educational Community License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.opensource.org/licenses/ECL-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * **********************************************************************************/ package org.sakaiproject.search.index.impl; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.nio.channels.FileChannel; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Date; import java.util.Iterator; import java.util.List; import javax.sql.DataSource; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.lucene.index.IndexReader; import org.apache.lucene.store.FSDirectory; import org.sakaiproject.search.api.SearchService; import org.sakaiproject.search.index.ClusterFilesystem; import org.sakaiproject.search.index.SegmentInfo; /** * This is a JDBC implementation of the ClusterFilesystem. It syncronizes the * local search index segments with the database, by sipping each segment and * pushing it to the database. Each Segment has an extra file that contains an * MD5 of the segment and a time stamp of the last update. If any segments are * missing in the local segment store (including no segments at all, as with a * new cluster node) the missing segments are downloaded from the JDBC store. If * any of the segments on the local store are found to be dammaged they are * reloaded from the database. * * @author ieb */ public class JDBCClusterIndexStore implements ClusterFilesystem { private static final Log log = LogFactory.getLog(JDBCClusterIndexStore.class); private DataSource dataSource = null; private String searchIndexDirectory = null; private static final String TEMP_INDEX_NAME = "tempindex"; private static final String INDEX_PATCHNAME = "indexpatch"; private static final long MAX_BLOCK_SIZE = 1024 * 1024 * 10; // 10M // blocks private boolean autoDdl = false; private boolean parallelIndex = false; /** * If validate is true, all segments will be checked on initial startup and * upload. This can take a long time. If its false, only when an index is * updated is the MD5 checked. Recomendation is to leave this false. */ private boolean validate = false; private String sharedSegments = null; private boolean debug = false; /** * locatStructuredStorage causes local segments to be placed into structured * storage on the local disk */ private boolean localStructuredStorage = false; /** * sharedStructuredStorage causes the shared segments to be placed into * shared structured storage in the shared location */ private boolean sharedStructuredStorage = false; private ClusterSegmentsStorage clusterStorage = null; private boolean localSegmentsOnly = false; private SearchService searchService; public void init() { try { log.info(this + ":init() "); clusterStorage = new ClusterSegmentsStorage(searchService, searchIndexDirectory, this, localStructuredStorage, debug); // We should migrate to the correct storage format, on the // local // and shared space, by looking at the DB and then checking what is // there // followed by a move. // Since we are doing a move, it should be ok to have this happend // on // the fly. try { migrateLocalSegments(); migrateSharedSegments(); } catch (IOException ex) { log.error( "Failed to migrate search content to new format, the instance should not continue to run ", ex); System.exit(-1); } /* * The storage is created by hibernate now try { if (autoDdl) { * SqlService.getInstance().ddl(this.getClass().getClassLoader(), * "search_cluster"); } } catch (Exception ex) { log.error("Failed * to init JDBCClusterIndexStorage", ex); } */ log.info(this + ":init() Ok "); } catch (Exception ex) { log.error("Failed to start Cluster Index store", ex); System.exit(-1); } } /** * Ther might need to be some locking here. When readers use this, they will * update the local segments with more information, update the local * Segmetns from the DB * * @param locked * A locak has been taken on the index */ public List<SegmentInfo> updateSegments() { Connection connection = null; List<SegmentInfo> segmentList = new ArrayList<SegmentInfo>(); try { connection = dataSource.getConnection(); List<SegmentInfo> dbSegments = getDBSegments(connection); if (log.isDebugEnabled()) log.debug("Update: DB Segments = " + dbSegments.size()); // remove files not in the dbSegmentList List<SegmentInfo> localSegments = getLocalSegments(); List<SegmentInfo> badLocalSegments = getBadLocalSegments(); // delete any bad local segments before we load so that they get // updated // from the db deleteAllSegments(badLocalSegments); List<SegmentInfo> deletedSegments = getDeletedLocalSegments(); // delete any segments marked as for deletion, by the last cycle. // if this is due to a index reader event, we should not be // performing this operation // as the current reader will have these files open. // we should only delete the old segments update thread // If this call forms part of an exception, then we should look at // doing a // timeout on the delete files. deleteAllSegments(deletedSegments); if (log.isDebugEnabled()) log.debug("Update: Local Segments = " + localSegments.size()); // which of the dbSegments are not present locally List<SegmentInfo> updateLocalSegments = new ArrayList<SegmentInfo>(); for (Iterator<SegmentInfo> i = dbSegments.iterator(); i.hasNext();) { SegmentInfo db_si = (SegmentInfo) i.next(); boolean found = false; String name = db_si.getName(); for (Iterator<SegmentInfo> j = localSegments.iterator(); j.hasNext();) { SegmentInfo local_si = (SegmentInfo) j.next(); if (name.equals(local_si.getName())) { found = true; break; } } if (!found) { updateLocalSegments.add(db_si); if (log.isDebugEnabled()) log.debug("Missing Will update " + db_si); } else { if (log.isDebugEnabled()) log.debug("Present Will Not update " + db_si); } } // which of the dbsegmetnts are newer than local versions for (Iterator<SegmentInfo> i = localSegments.iterator(); i.hasNext();) { SegmentInfo current_si = (SegmentInfo) i.next(); boolean found = false; String name = current_si.getName(); long version = current_si.getVersion(); for (Iterator<SegmentInfo> j = dbSegments.iterator(); j.hasNext();) { SegmentInfo db_si = (SegmentInfo) j.next(); if (name.equals(db_si.getName()) && db_si.getVersion() > version) { updateLocalSegments.add(db_si); if (log.isDebugEnabled()) log.debug("Newer will Update " + db_si); found = true; break; } } if (!found) { if (log.isDebugEnabled()) log.debug("Ok will not update " + current_si); } } List<SegmentInfo> removeLocalSegments = new ArrayList<SegmentInfo>(); // which segments exist locally but not in the DB, these should // be // removed for (Iterator<SegmentInfo> i = localSegments.iterator(); i.hasNext();) { SegmentInfo local_si = (SegmentInfo) i.next(); // only check local segments that are not new and not if (local_si.isCreated()) { boolean found = false; String name = local_si.getName(); for (Iterator<SegmentInfo> j = dbSegments.iterator(); j.hasNext();) { SegmentInfo db_si = (SegmentInfo) j.next(); if (name.equals(db_si.getName())) { found = true; break; } } if (!found) { removeLocalSegments.add(local_si); if (log.isDebugEnabled()) log.debug("Will remove " + local_si); } else { if (log.isDebugEnabled()) log.debug("Ok Will not remove " + local_si); } } } // if we could mark the local segment for deletion so that // its is only deleted the next time a lock is taken on the index for (Iterator<SegmentInfo> i = removeLocalSegments.iterator(); i.hasNext();) { SegmentInfo rmsi = (SegmentInfo) i.next(); removeLocalSegment(rmsi); } // process the get list, first markign the segments as existing, so // that other threads // dont update them, then perform the update. try { for (Iterator<SegmentInfo> i = updateLocalSegments.iterator(); i.hasNext();) { SegmentInfo addsi = (SegmentInfo) i.next(); addsi.lockLocalSegment(); } for (Iterator<SegmentInfo> i = updateLocalSegments.iterator(); i.hasNext();) { SegmentInfo addsi = (SegmentInfo) i.next(); try { // This is thread safe strangely since the source doesnt // change // hence although it could be wastefull, more than one // copy can perform an update at a // time. if (addsi.isLocalLock()) { updateLocalSegment(connection, addsi); } else { log.warn("Not Updating Segment, since lock is not on this thread " + addsi.getName()); } } catch (Exception ex) { // ignore failures to unpack a local segment. It may // have // been removed by // annother node log.info("Segment was not unpacked " + ex.getClass().getName() + ":" + ex.getMessage()); } } } finally { for (Iterator<SegmentInfo> i = updateLocalSegments.iterator(); i.hasNext();) { SegmentInfo addsi = (SegmentInfo) i.next(); addsi.unlockLocalSegment(); } } // if we made any modifications, we also need to process the patch if (updateLocalSegments.size() > 0) { updateLocalPatch(connection); } // build the list putting the current segment at the end for (Iterator<SegmentInfo> i = dbSegments.iterator(); i.hasNext();) { SegmentInfo si = (SegmentInfo) i.next(); File f = si.getSegmentLocation(); if (f.exists()) { // only add those segments that exist after the sync segmentList.add(si); } if (log.isDebugEnabled()) log.debug("Segment Present at " + f.getName()); } connection.commit(); } catch (Exception sqle) { log.error("Failed to update segments ", sqle); try { connection.rollback(); } catch (Exception ex) { log.debug(ex); } } finally { try { connection.close(); } catch (Exception e) { log.debug(e); } } return segmentList; } private void deleteAllSegments(List<SegmentInfo> badLocalSegments) { for (Iterator<SegmentInfo> i = badLocalSegments.iterator(); i.hasNext();) { SegmentInfo s = i.next(); s.doFinalDelete(); } } /** * save the local segments to the DB */ public List<SegmentInfo> saveSegments() { Connection connection = null; List<SegmentInfo> segmentList = new ArrayList<SegmentInfo>(); try { connection = dataSource.getConnection(); List<SegmentInfo> dbSegments = getDBSegments(connection); // remove files not in the dbSegmentList List<SegmentInfo> localSegments = getLocalSegments(); List<SegmentInfo> badLocalSegments = getBadLocalSegments(); // find the dbSegments that are not present locally List<SegmentInfo> removeDBSegments = new ArrayList<SegmentInfo>(); List<SegmentInfo> currentDBSegments = new ArrayList<SegmentInfo>(); // which segments exist inthe db but not locally for (Iterator<SegmentInfo> i = dbSegments.iterator(); i.hasNext();) { SegmentInfo db_si = (SegmentInfo) i.next(); boolean found = false; String name = db_si.getName(); for (Iterator<SegmentInfo> j = localSegments.iterator(); j.hasNext();) { SegmentInfo local_si = (SegmentInfo) j.next(); if (name.equals(local_si.getName())) { found = true; break; } } // dont delete bad segments from the DB if (!found) { for (Iterator<SegmentInfo> j = badLocalSegments.iterator(); j.hasNext();) { SegmentInfo local_file = (SegmentInfo) j.next(); if (name.equals(local_file.getName())) { found = true; break; } } } if (!found) { removeDBSegments.add(db_si); if (log.isDebugEnabled()) log.debug("Will remove from the DB " + db_si); } else { currentDBSegments.add(db_si); if (log.isDebugEnabled()) log.debug("In the DB will not remove " + db_si); } } List<SegmentInfo> updateDBSegments = new ArrayList<SegmentInfo>(); // which of the localSegments are not in the db for (Iterator<SegmentInfo> i = localSegments.iterator(); i.hasNext();) { SegmentInfo local_si = (SegmentInfo) i.next(); boolean found = false; String name = local_si.getName(); for (Iterator<SegmentInfo> j = dbSegments.iterator(); j.hasNext();) { SegmentInfo db_si = (SegmentInfo) j.next(); if (name.equals(db_si.getName())) { found = true; break; } } if (!found) { updateDBSegments.add(local_si); if (log.isDebugEnabled()) log.debug(" Will update to the DB " + local_si); } else { if (log.isDebugEnabled()) log.debug(" Will NOT update to the DB " + local_si); } } // which of the localSegments have been modified for (Iterator<SegmentInfo> i = localSegments.iterator(); i.hasNext();) { SegmentInfo local_si = (SegmentInfo) i.next(); boolean found = false; String name = local_si.getName(); long version = local_si.getVersion(); for (Iterator<SegmentInfo> j = dbSegments.iterator(); j.hasNext();) { SegmentInfo db_si = (SegmentInfo) j.next(); if (name.equals(db_si.getName()) && version > db_si.getVersion()) { updateDBSegments.add(db_si); if (log.isDebugEnabled()) log.debug("Will update modified to the DB " + db_si); found = true; break; } } if (!found) { if (log.isDebugEnabled()) log.debug("Will not update the DB, matches " + local_si); } } // process the get list for (Iterator<SegmentInfo> i = updateDBSegments.iterator(); i.hasNext();) { SegmentInfo addsi = (SegmentInfo) i.next(); updateDBSegment(connection, addsi); } // build the list putting the current segment at the end updateDBPatch(connection); for (Iterator<SegmentInfo> i = updateDBSegments.iterator(); i.hasNext();) { SegmentInfo si = (SegmentInfo) i.next(); File f = si.getSegmentLocation(); segmentList.add(si); if (log.isDebugEnabled()) log.debug("Segments saved " + f.getName()); } // process the remove list, the update was Ok so we can remove all // the old segments for (Iterator<SegmentInfo> i = removeDBSegments.iterator(); i.hasNext();) { SegmentInfo rmsi = (SegmentInfo) i.next(); removeDBSegment(connection, rmsi); } connection.commit(); deleteAllSegments(badLocalSegments); } catch (Exception ex) { log.error("Failed to Save Segments back to Central Storage", ex); try { connection.rollback(); } catch (Exception e) { log.debug(e); } recoverFromFailure(); } finally { try { connection.close(); } catch (Exception e) { log.debug(e); } } return segmentList; } public List<SegmentInfo> saveAllSegments() { Connection connection = null; List<SegmentInfo> segmentList = new ArrayList<SegmentInfo>(); try { connection = dataSource.getConnection(); List<SegmentInfo> dbSegments = getDBSegments(connection); // remove files not in the dbSegmentList List<SegmentInfo> localSegments = getLocalSegments(); List<SegmentInfo> badLocalSegments = getBadLocalSegments(); // find the dbSegments that are not present locally List<SegmentInfo> updateDBSegments = new ArrayList<SegmentInfo>(); // which of the localSegments are not in the db for (Iterator<SegmentInfo> i = localSegments.iterator(); i.hasNext();) { SegmentInfo local_si = (SegmentInfo) i.next(); boolean found = false; String name = local_si.getName(); for (Iterator<SegmentInfo> j = dbSegments.iterator(); j.hasNext();) { SegmentInfo db_si = (SegmentInfo) j.next(); if (name.equals(db_si.getName())) { found = true; break; } } // dont delete bad segments from the DB if (!found) { for (Iterator<SegmentInfo> j = badLocalSegments.iterator(); j.hasNext();) { SegmentInfo local_file = (SegmentInfo) j.next(); if (name.equals(local_file.getName())) { found = true; break; } } } if (!found) { updateDBSegments.add(local_si); } } // the db segments for (Iterator<SegmentInfo> i = localSegments.iterator(); i.hasNext();) { SegmentInfo local_si = (SegmentInfo) i.next(); String name = local_si.getName(); for (Iterator<SegmentInfo> j = dbSegments.iterator(); j.hasNext();) { SegmentInfo db_si = (SegmentInfo) j.next(); if (name.equals(db_si.getName())) { updateDBSegments.add(db_si); break; } } } // process the get list for (Iterator<SegmentInfo> i = updateDBSegments.iterator(); i.hasNext();) { SegmentInfo addsi = (SegmentInfo) i.next(); updateDBSegment(connection, addsi); } // build the list putting the current segment at the end for (Iterator<SegmentInfo> i = updateDBSegments.iterator(); i.hasNext();) { SegmentInfo si = (SegmentInfo) i.next(); segmentList.add(si); } connection.commit(); deleteAllSegments(badLocalSegments); } catch (Exception ex) { log.error("Failed to Save Segments back to Central Storage", ex); try { connection.rollback(); } catch (Exception e) { log.debug(e); } recoverFromFailure(); } finally { try { connection.close(); } catch (Exception e) { log.debug(e); } } return segmentList; } protected void updateLocalSegment(Connection connection, SegmentInfo addsi) throws SQLException, IOException { if (searchService.hasDiagnostics()) { log.info("\tUpdate Local Segment from Database " + addsi); } if (localSegmentsOnly) { log.warn("Update Local Segment Requested with inactive Shared Storage " + addsi); } else { if (sharedSegments == null || sharedSegments.length() == 0) { updateLocalSegmentBLOB(connection, addsi); } else { updateLocalSegmentFilesystem(connection, addsi); } } } /** * updte a segment from the database * * @param connection * @param addsi */ protected void updateLocalSegmentBLOB(Connection connection, SegmentInfo addsi) throws SQLException, IOException { if (log.isDebugEnabled()) log.debug("Updating local segment from databse " + addsi); PreparedStatement segmentSelect = null; ResultSet resultSet = null; try { segmentSelect = connection .prepareStatement("select version_, packet_ from search_segments where name_ = ?"); segmentSelect.clearParameters(); segmentSelect.setString(1, addsi.getName()); resultSet = segmentSelect.executeQuery(); if (resultSet.next()) { InputStream packetStream = null; try { long version = resultSet.getLong(1); packetStream = resultSet.getBinaryStream(2); clusterStorage.unpackSegment(addsi, packetStream, version); if (log.isDebugEnabled()) log.debug("Updated Packet from DB to versiob " + version); } finally { try { packetStream.close(); } catch (Exception ex) { log.debug(ex); } } } else { log.error("Didnt find segment in database"); } } finally { try { resultSet.close(); } catch (Exception ex) { log.debug(ex); } try { segmentSelect.close(); } catch (Exception ex) { log.debug(ex); } } } /** * remove a local segment * * @param rmsi */ public void removeLocalSegment(SegmentInfo rmsi) { rmsi.setDeleted(); if (searchService.hasDiagnostics()) { log.info("\tMarked Local Segment for deletion " + rmsi); } } /** * get a list of all DB segments ordered by version * * @param connection * @return */ private List<SegmentInfo> getDBSegments(Connection connection) throws SQLException { PreparedStatement segmentAllSelect = null; ResultSet resultSet = null; List<SegmentInfo> dbsegments = new ArrayList<SegmentInfo>(); try { segmentAllSelect = connection .prepareStatement("select version_, name_ from search_segments where name_ <> ? "); segmentAllSelect.clearParameters(); segmentAllSelect.setString(1, INDEX_PATCHNAME); resultSet = segmentAllSelect.executeQuery(); while (resultSet.next()) { final long version = resultSet.getLong(1); final String name = resultSet.getString(2); SegmentInfo si = SegmentInfoImpl.newSharedSegmentInfo(name, version, localStructuredStorage, searchIndexDirectory); dbsegments.add(si); if (log.isDebugEnabled()) log.debug("DB Segment " + si); } } finally { try { resultSet.close(); } catch (Exception ex) { log.debug(ex); } try { segmentAllSelect.close(); } catch (Exception ex) { log.debug(ex); } } return dbsegments; } protected void updateDBPatch(Connection connection) throws SQLException, IOException { if (localSegmentsOnly) { if (log.isDebugEnabled()) log.debug("Update Patch Requested with inactive Shared Storage "); } else { if (sharedSegments == null || sharedSegments.length() == 0) { updateDBPatchBLOB(connection); } else { updateDBPatchFilesystem(connection); } } } /** * updat this save this local segment into the db * * @param connection * @param addsi */ protected void updateDBPatchBLOB(Connection connection) throws SQLException, IOException { PreparedStatement segmentUpdate = null; PreparedStatement segmentInsert = null; InputStream packetStream = null; File packetFile = null; long newVersion = System.currentTimeMillis(); try { segmentUpdate = connection.prepareStatement( "update search_segments set packet_ = ?, version_ = ?, size_ = ? where name_ = ?"); segmentInsert = connection.prepareStatement( "insert into search_segments (packet_, name_, version_, size_ ) values ( ?,?,?,?)"); packetFile = clusterStorage.packPatch(); if (packetFile.exists()) { packetStream = new FileInputStream(packetFile); segmentUpdate.clearParameters(); segmentUpdate.setBinaryStream(1, packetStream, (int) packetFile.length()); segmentUpdate.setLong(2, newVersion); segmentUpdate.setLong(3, packetFile.length()); segmentUpdate.setString(4, INDEX_PATCHNAME); if (segmentUpdate.executeUpdate() != 1) { segmentInsert.clearParameters(); segmentInsert.setBinaryStream(1, packetStream, (int) packetFile.length()); segmentInsert.setString(2, INDEX_PATCHNAME); segmentInsert.setLong(3, newVersion); segmentInsert.setLong(4, packetFile.length()); if (segmentInsert.executeUpdate() != 1) { throw new SQLException(" Failed to insert patch "); } } if (log.isDebugEnabled()) log.debug("DB Updated Patch "); } else { log.warn(" Packed Patch does not exist " + packetFile.getPath()); } } finally { try { if (packetStream != null) { packetStream.close(); } } catch (Exception ex) { log.debug(ex); } try { packetFile.delete(); } catch (Exception ex) { log.debug(ex); } try { segmentUpdate.close(); } catch (Exception ex) { log.debug(ex); } try { segmentInsert.close(); } catch (Exception ex) { log.debug(ex); } } } /** * updat this save this local segment into the db * * @param connection * @param addsi */ protected void updateDBPatchFilesystem(Connection connection) throws SQLException, IOException { PreparedStatement segmentUpdate = null; PreparedStatement segmentInsert = null; FileChannel packetStream = null; FileInputStream packetFIS = null; FileChannel sharedStream = null; FileOutputStream sharedFOS = null; File packetFile = null; File sharedFinalFile = null; File sharedTempFile = null; long newVersion = System.currentTimeMillis(); try { sharedTempFile = new File(getSharedTempFileName(INDEX_PATCHNAME)); sharedFinalFile = new File(getSharedFileName(INDEX_PATCHNAME, sharedStructuredStorage)); packetFile = clusterStorage.packPatch(); if (packetFile.exists()) { packetFIS = new FileInputStream(packetFile); packetStream = packetFIS.getChannel(); File sharedTempFileParent = sharedTempFile.getParentFile(); if (!sharedTempFileParent.exists() && !sharedTempFileParent.mkdirs()) { log.warn("couldn't create " + sharedTempFileParent.getPath()); } sharedFOS = new FileOutputStream(sharedTempFile); sharedStream = sharedFOS.getChannel(); doBlockedStream(packetStream, sharedStream); packetStream.close(); sharedStream.close(); segmentUpdate = connection .prepareStatement("update search_segments set version_ = ?, size_ = ? where name_ = ? "); segmentInsert = connection .prepareStatement("insert into search_segments ( name_, version_, size_ ) values ( ?,?,?)"); segmentUpdate.clearParameters(); segmentUpdate.setLong(1, newVersion); segmentUpdate.setLong(2, packetFile.length()); segmentUpdate.setString(3, INDEX_PATCHNAME); if (segmentUpdate.executeUpdate() != 1) { segmentInsert.clearParameters(); segmentInsert.setString(1, INDEX_PATCHNAME); segmentInsert.setLong(2, newVersion); segmentInsert.setLong(3, packetFile.length()); if (segmentInsert.executeUpdate() != 1) { throw new SQLException(" Failed to add patch packet "); } } long st = System.currentTimeMillis(); if (!sharedTempFile.renameTo(sharedFinalFile)) { log.warn("Couldn't rename file " + sharedTempFile.getPath() + " to " + sharedFinalFile.getPath()); } if (searchService.hasDiagnostics()) { log.info("Renamed " + sharedTempFile.getPath() + " to " + sharedFinalFile.getPath() + " in " + (System.currentTimeMillis() - st) + "ms"); } } else { log.warn("Packet file does not exist " + packetFile.getPath()); } } finally { try { if (packetStream != null) { packetStream.close(); packetFIS.close(); } } catch (Exception ex) { log.debug(ex); } try { packetFile.delete(); } catch (Exception ex) { log.debug(ex); } try { if (sharedStream != null) { sharedStream.close(); sharedFOS.close(); } } catch (Exception ex) { log.debug(ex); } try { sharedTempFile.delete(); } catch (Exception ex) { log.debug(ex); } try { segmentUpdate.close(); } catch (Exception ex) { log.debug(ex); } try { segmentInsert.close(); } catch (Exception ex) { log.debug(ex); } } } protected void updateDBSegment(Connection connection, SegmentInfo addsi) throws SQLException, IOException { if (searchService.hasDiagnostics()) { log.info("\tUpdate Database Segment from Local " + addsi); } if (localSegmentsOnly) { if (log.isDebugEnabled()) log.debug("Not Saving Segment to DB as no Shared Storage " + addsi); } else { if (sharedSegments == null || sharedSegments.length() == 0) { updateDBSegmentBLOB(connection, addsi); } else { updateDBSegmentFilesystem(connection, addsi); } } } /** * updat this save this local segment into the db * * @param connection * @param addsi */ protected void updateDBSegmentBLOB(Connection connection, SegmentInfo addsi) throws SQLException, IOException { PreparedStatement segmentUpdate = null; PreparedStatement segmentInsert = null; InputStream packetStream = null; File packetFile = null; long newVersion = System.currentTimeMillis(); try { segmentUpdate = connection.prepareStatement( "update search_segments set packet_ = ?, version_ = ?, size_ = ? where name_ = ? and version_ = ?"); segmentInsert = connection.prepareStatement( "insert into search_segments (packet_, name_, version_, size_ ) values ( ?,?,?,?)"); packetFile = clusterStorage.packSegment(addsi, newVersion); if (packetFile.exists()) { packetStream = new FileInputStream(packetFile); if (addsi.isInDb()) { segmentUpdate.clearParameters(); segmentUpdate.setBinaryStream(1, packetStream, (int) packetFile.length()); segmentUpdate.setLong(2, newVersion); segmentUpdate.setLong(3, packetFile.length()); segmentUpdate.setString(4, addsi.getName()); segmentUpdate.setLong(5, addsi.getVersion()); if (segmentUpdate.executeUpdate() != 1) { throw new SQLException(" ant Find packet to update " + addsi); } } else { segmentInsert.clearParameters(); segmentInsert.setBinaryStream(1, packetStream, (int) packetFile.length()); segmentInsert.setString(2, addsi.getName()); segmentInsert.setLong(3, newVersion); segmentInsert.setLong(4, packetFile.length()); if (segmentInsert.executeUpdate() != 1) { throw new SQLException(" Failed to insert packet " + addsi); } } addsi.setVersion(newVersion); if (log.isDebugEnabled()) log.debug("DB Updated " + addsi); try { packetStream.close(); } catch (Exception ex) { log.debug(ex); } try { packetFile.delete(); } catch (Exception ex) { log.debug(ex); } } else { log.warn("Packet file does not exist " + packetFile.getPath()); } } finally { try { packetStream.close(); } catch (Exception ex) { log.debug(ex); } try { packetFile.delete(); } catch (Exception ex) { log.debug(ex); } try { segmentUpdate.close(); } catch (Exception ex) { log.debug(ex); } try { segmentInsert.close(); } catch (Exception ex) { log.debug(ex); } } } private void removeDBSegment(Connection connection, SegmentInfo rmsi) throws SQLException { PreparedStatement segmentDelete = null; try { if (rmsi.isInDb()) { segmentDelete = connection .prepareStatement("delete from search_segments where name_ = ? and version_ = ?"); segmentDelete.clearParameters(); segmentDelete.setString(1, rmsi.getName()); segmentDelete.setLong(2, rmsi.getVersion()); segmentDelete.execute(); String sharedSegment = getSharedFileName(rmsi.getName(), sharedStructuredStorage); if (sharedSegment != null) { File f = new File(sharedSegment); if (f.exists()) { if (!f.delete()) { log.warn("unable to delete " + f.getPath()); } } } if (searchService.hasDiagnostics()) { log.info("\tRemoved Segment From Database [" + rmsi + "]"); } } } finally { try { if (segmentDelete != null) { segmentDelete.close(); } } catch (Exception ex) { log.debug(ex); } } } /** * create a new local segment and mark its tiestamp */ public SegmentInfo newSegment() throws IOException { File f = null; for (;;) { f = SegmentInfoImpl.getSegmentLocation(String.valueOf(System.currentTimeMillis()), localStructuredStorage, searchIndexDirectory); if (!f.exists()) { break; } } if (!f.mkdirs()) { log.warn("unable to create directory: " + f.getPath()); } SegmentInfo si = SegmentInfoImpl.newLocalSegmentInfo(f, localStructuredStorage, searchIndexDirectory); si.setNew(); si.setTimeStamp(System.currentTimeMillis()); return si; } /** * get a list of local segments * * @return * @throws IOException */ public List<SegmentInfo> getLocalSegments() throws IOException { List<SegmentInfo> l = new ArrayList<SegmentInfo>(); File searchDir = new File(searchIndexDirectory); return getLocalSegments(searchDir, l); } /** * recurse into a list of segments * * @param searchDir * @param l * @return * @throws IOException */ public List<SegmentInfo> getLocalSegments(File searchDir, List<SegmentInfo> l) throws IOException { File[] files = searchDir.listFiles(); if (files != null) { for (int i = 0; i < files.length; i++) { if (files[i].isDirectory()) { SegmentInfo sgi = SegmentInfoImpl.newLocalSegmentInfo(files[i], localStructuredStorage, searchIndexDirectory); if (sgi.isClusterSegment()) { if (IndexReader.indexExists(FSDirectory.open(files[i]))) { if (sgi.isCreated()) { l.add(sgi); if (log.isDebugEnabled()) log.debug("LO Segment " + sgi); } else { if (log.isDebugEnabled()) log.debug("LO Segment not created " + sgi.toString()); } } else { log.warn("Found Orphaned directory with no segment information present " + files[i]); } } else { l = getLocalSegments(files[i], l); } } } } return l; } /** * get a list of bad segmetns with brokenindexes * * @return * @throws IOException */ private List<SegmentInfo> getBadLocalSegments() throws IOException { List<SegmentInfo> l = new ArrayList<SegmentInfo>(); File searchDir = new File(searchIndexDirectory); return getBadLocalSegments(searchDir, l); } /** * get a list of segments that are ready for deletion * * @return * @throws IOException */ private List<SegmentInfo> getDeletedLocalSegments() throws IOException { List<SegmentInfo> l = new ArrayList<SegmentInfo>(); File searchDir = new File(searchIndexDirectory); return getDeletedLocalSegments(searchDir, l); } /** * recurse into a list of bad local segments * * @param searchDir * @param l * @return * @throws IOException */ private List<SegmentInfo> getBadLocalSegments(File searchDir, List<SegmentInfo> l) throws IOException { if (searchDir.isDirectory()) { File[] files = searchDir.listFiles(); if (files != null) { for (int i = 0; i < files.length; i++) { SegmentInfo sgi = SegmentInfoImpl.newLocalSegmentInfo(files[i], localStructuredStorage, searchIndexDirectory); if (sgi.isClusterSegment()) { if (sgi.isCreated()) { if (!IndexReader.indexExists(FSDirectory.open(files[i]))) { l.add(sgi); } } } else { l = getBadLocalSegments(files[i], l); } } } } return l; } /** * Get a list of segments to be deleted * * @param searchDir * @param l * @return * @throws IOException */ private List<SegmentInfo> getDeletedLocalSegments(File searchDir, List<SegmentInfo> l) throws IOException { if (searchDir.isDirectory()) { File[] files = searchDir.listFiles(); if (files != null) { for (int i = 0; i < files.length; i++) { SegmentInfo sgi = SegmentInfoImpl.newLocalSegmentInfo(files[i], localStructuredStorage, searchIndexDirectory); if (sgi.isClusterSegment()) { if (sgi.isDeleted()) { l.add(sgi); } } else { l = getDeletedLocalSegments(files[i], l); } } } } return l; } /** * recover from a failiure */ private void recoverFromFailure() { log.error("Recover from Failiure is not implementated at the moment," + " the local index is corrupt, please delete it and it will " + "reload from the database"); } /** * @return Returns the dataSource. */ public DataSource getDataSource() { return dataSource; } /** * @param dataSource * The dataSource to set. */ public void setDataSource(DataSource dataSource) { this.dataSource = dataSource; } public void setLocation(String location) { searchIndexDirectory = location; log.info("Search Index Location is " + location); } /** * @param autoDdl * The autoDdl to set. */ public void setAutoDdl(boolean autoDdl) { this.autoDdl = autoDdl; } /** * create a temporary index for indexing operations */ public File getTemporarySegment(boolean delete) { // this index will not have a timestamp, and hence will not be part sync // with the db File f = new File(searchIndexDirectory, TEMP_INDEX_NAME); if (delete && f.exists()) { SegmentInfoImpl.deleteAll(f); } if (!f.exists() && !f.mkdirs()) { log.warn("couldn't create directory " + f.getPath()); } return f; } public void removeTemporarySegment() { File f = new File(searchIndexDirectory, TEMP_INDEX_NAME); if (f.exists()) { SegmentInfoImpl.deleteAll(f); } } public SegmentInfo saveTemporarySegment() throws IOException { SegmentInfo segInfo = newSegment(); File s = new File(searchIndexDirectory, TEMP_INDEX_NAME); File d = segInfo.getSegmentLocation(); copyAll(s, d); segInfo.setCreated(); segInfo.touchSegment(); return segInfo; } /** * Copy a file to a directory, if the source is a directory, the copy * recurses into the directory. If the destination does not exists, it is * assumed to be a file. * * @param s * the source file * @param d * the source directory * @throws IOException */ private void copyAll(File s, File d) throws IOException { if (s.isDirectory()) { File[] fl = s.listFiles(); for (int i = 0; i < fl.length; i++) { if (fl[i].isFile()) { copyFile(fl[i], d); } else { File nd = new File(d, fl[i].getName()); if (!nd.mkdirs()) { log.warn("couldn't create directories " + nd.getPath()); } copyAll(fl[i], nd); } } } else { copyFile(s, d); } } /** * Copy a file from s to d, s will be a file, d may be a file or directory * * @param s * @param d * @throws IOException */ private void copyFile(File s, File d) throws IOException { if (log.isDebugEnabled()) log.debug("Copying " + s.getAbsolutePath() + " to " + d.getAbsolutePath()); if (s.exists() && s.isFile()) { File t = d; // target if (d.isDirectory()) { if (!d.exists()) { if (!d.mkdirs()) { log.warn("Unable to create directory " + d.getPath()); } } t = new File(d, s.getName()); } else { File p = d.getParentFile(); if (!p.exists()) { if (!p.mkdirs()) { log.warn("couldn't create: " + p.getPath()); } } } FileChannel srcChannel = null; FileChannel dstChannel = null; try { // use nio // Create channel on the source srcChannel = new FileInputStream(s).getChannel(); // Create channel on the destination dstChannel = new FileOutputStream(t).getChannel(); // Copy file contents from source to destination doBlockedStream(srcChannel, dstChannel); // Close the channels } finally { try { srcChannel.close(); } catch (Exception ex) { log.debug(ex); } try { dstChannel.close(); } catch (Exception ex) { log.debug(ex); } } } } public void recoverSegment(SegmentInfo recoverSegInfo) { SegmentInfo segInfo = SegmentInfoImpl.newLocalSegmentInfo(recoverSegInfo); segInfo.debugSegment("Pre Recovery Check :"); segInfo.compareTo("Comparing Disk Segment to Recovered Segment", recoverSegInfo); recoverSegInfo.setDeleted(); recoverSegInfo.doFinalDelete(); recoverSegInfo.setNew(); Connection connection = null; try { connection = dataSource.getConnection(); updateLocalSegment(connection, recoverSegInfo); // we also need to re-apply the patch updateLocalPatch(connection); connection.commit(); } catch (Exception ex) { try { if (connection != null) { connection.rollback(); } } catch (Exception e) { log.debug(ex); } throw new RuntimeException("Failed to recover dammaged segment ", ex); } finally { try { if (connection != null) { connection.close(); } } catch (Exception e) { log.debug(e); } } SegmentInfo newSegInfo = SegmentInfoImpl.newLocalSegmentInfo(recoverSegInfo); newSegInfo.debugSegment("Recovered Segment"); newSegInfo.compareTo("Comparing Recoverd Segment to Previous Disk Segment", segInfo); } protected void updateLocalPatch(Connection connection) throws SQLException, IOException { if (localSegmentsOnly) { log.warn("Update Patch Requested with inactive Shared Storage "); } else { if (sharedSegments == null || sharedSegments.length() == 0) { updateLocalPatchBLOB(connection); } else { updateLocalPatchFilesystem(connection); } } } protected void updateLocalPatchFilesystem(Connection connection) throws SQLException, IOException { if (log.isDebugEnabled()) log.debug("Updating local patch "); PreparedStatement segmentSelect = null; ResultSet resultSet = null; try { segmentSelect = connection.prepareStatement("select version_ from search_segments where name_ = ?"); segmentSelect.clearParameters(); segmentSelect.setString(1, INDEX_PATCHNAME); resultSet = segmentSelect.executeQuery(); if (resultSet.next()) { InputStream packetStream = null; try { File f = new File(getSharedFileName(INDEX_PATCHNAME, sharedStructuredStorage)); if (f.exists()) { packetStream = new FileInputStream(f); clusterStorage.unpackPatch(packetStream); if (log.isDebugEnabled()) log.debug("Updated Patch "); } else { log.warn("Shared Segment File does not exist " + f.getPath()); } } finally { try { if (packetStream != null) { packetStream.close(); } } catch (Exception ex) { log.debug(ex); } } } else { if (log.isDebugEnabled()) log.debug("Didnt find patch in database, this is Ok"); } } finally { try { resultSet.close(); } catch (Exception ex) { log.debug(ex); } try { segmentSelect.close(); } catch (Exception ex) { log.debug(ex); } } } private void updateLocalPatchBLOB(Connection connection) throws SQLException, IOException { if (log.isDebugEnabled()) log.debug("Updating local patch "); PreparedStatement segmentSelect = null; ResultSet resultSet = null; try { segmentSelect = connection .prepareStatement("select version_, packet_ from search_segments where name_ = ?"); segmentSelect.clearParameters(); segmentSelect.setString(1, INDEX_PATCHNAME); resultSet = segmentSelect.executeQuery(); if (resultSet.next()) { InputStream packetStream = null; try { long version = resultSet.getLong(1); packetStream = resultSet.getBinaryStream(2); clusterStorage.unpackPatch(packetStream); if (log.isDebugEnabled()) log.debug("Updated Patch from DB " + version); } finally { try { packetStream.close(); } catch (Exception ex) { log.debug(ex); } } } else { if (log.isDebugEnabled()) log.debug("Didnt find patch in database, this is Ok "); } } finally { try { resultSet.close(); } catch (Exception ex) { log.debug(ex); } try { segmentSelect.close(); } catch (Exception ex) { log.debug(ex); } } } public String getSegmentName(String segmentPath) { File f = new File(segmentPath); return f.getName(); } /** * @return Returns the validate. */ public boolean isValidate() { return validate; } /** * @param validate * The validate to set. */ public void setValidate(boolean validate) { this.validate = validate; } /** * updat this save this local segment into the db * * @param connection * @param addsi */ protected void updateDBSegmentFilesystem(Connection connection, SegmentInfo addsi) throws SQLException, IOException { PreparedStatement segmentUpdate = null; PreparedStatement segmentInsert = null; FileChannel packetStream = null; FileInputStream packetFIS = null; FileChannel sharedStream = null; FileOutputStream sharedFOS = null; File packetFile = null; File sharedFinalFile = null; File sharedTempFile = null; long newVersion = System.currentTimeMillis(); try { sharedTempFile = new File(getSharedTempFileName(addsi.getName())); sharedFinalFile = new File(getSharedFileName(addsi.getName(), sharedStructuredStorage)); packetFile = clusterStorage.packSegment(addsi, newVersion); if (packetFile.exists()) { packetFIS = new FileInputStream(packetFile); packetStream = packetFIS.getChannel(); File parentFile = sharedTempFile.getParentFile(); if (!parentFile.exists() && !parentFile.mkdirs()) { log.warn("Unable to create directory " + sharedTempFile.getParentFile().getPath()); } sharedFOS = new FileOutputStream(sharedTempFile); sharedStream = sharedFOS.getChannel(); // Copy file contents from source to destination doBlockedStream(packetStream, sharedStream); packetStream.close(); sharedStream.close(); segmentUpdate = connection.prepareStatement( "update search_segments set version_ = ?, size_ = ? where name_ = ? and version_ = ?"); segmentInsert = connection .prepareStatement("insert into search_segments ( name_, version_, size_ ) values ( ?,?,?)"); if (addsi.isInDb()) { segmentUpdate.clearParameters(); segmentUpdate.setLong(1, newVersion); segmentUpdate.setLong(2, packetFile.length()); segmentUpdate.setString(3, addsi.getName()); segmentUpdate.setLong(4, addsi.getVersion()); if (segmentUpdate.executeUpdate() != 1) { throw new SQLException(" ant Find packet to update " + addsi); } } else { segmentInsert.clearParameters(); segmentInsert.setString(1, addsi.getName()); segmentInsert.setLong(2, newVersion); segmentInsert.setLong(3, packetFile.length()); if (segmentInsert.executeUpdate() != 1) { throw new SQLException(" Failed to insert packet " + addsi); } } addsi.setVersion(newVersion); File sharedParentFile = sharedFinalFile.getParentFile(); if (!sharedParentFile.exists() && !sharedParentFile.mkdirs()) { log.warn("Couln't create directory " + sharedParentFile.getPath()); } long st = System.currentTimeMillis(); if (!sharedTempFile.renameTo(sharedFinalFile)) { log.warn("Couldn't rename " + sharedTempFile.getPath() + " to " + sharedFinalFile.getPath()); } if (searchService.hasDiagnostics()) { log.info("Renamed " + sharedTempFile.getPath() + " to " + sharedFinalFile.getPath() + " in " + (System.currentTimeMillis() - st) + "ms"); } log.info("DB Updated " + addsi); } else { log.warn("Packet file does not exist " + packetFile.getPath()); } } finally { try { packetStream.close(); packetFIS.close(); } catch (Exception ex) { log.debug(ex); } try { packetFile.delete(); } catch (Exception ex) { log.debug(ex); } try { sharedStream.close(); sharedFOS.close(); } catch (Exception ex) { log.debug(ex); } try { sharedTempFile.delete(); } catch (Exception ex) { log.debug(ex); } try { segmentUpdate.close(); } catch (Exception ex) { log.debug(ex); } try { segmentInsert.close(); } catch (Exception ex) { log.debug(ex); } } } /** * @param packetStream * @param sharedStream * @throws IOException */ private void doBlockedStream(FileChannel from, FileChannel to) throws IOException { to.position(0); long size = from.size(); for (long pos = 0; pos < size;) { long count = size - pos; if (count > MAX_BLOCK_SIZE) { count = MAX_BLOCK_SIZE; } to.position(pos); long cpos = to.position(); log.debug("NIOTransfering |" + count + "| bytes from |" + pos + "| to |" + cpos + "|"); long t = to.transferFrom(from, pos, count); pos = pos + t; } log.debug(" Final Size Source " + from.size()); log.debug(" Final Size Destination " + to.size()); } private String getSharedFileName(String name, boolean structured) { if (localSegmentsOnly) { return null; } if (sharedSegments != null && sharedSegments.length() > 0) { if (!sharedSegments.endsWith("/")) { sharedSegments = sharedSegments + "/"; } if (structured && !INDEX_PATCHNAME.equals(name)) { String hashName = name.substring(name.length() - 4, name.length() - 2); return sharedSegments + hashName + "/" + name + ".zip"; } else { return sharedSegments + name + ".zip"; } } return null; } private String getSharedTempFileName(String name) { if (sharedSegments != null && sharedSegments.length() > 0) { if (!sharedSegments.endsWith("/")) { sharedSegments = sharedSegments + "/"; } return sharedSegments + name + ".zip." + System.currentTimeMillis(); } return null; } /** * updte a segment from the database * * @param connection * @param addsi */ protected void updateLocalSegmentFilesystem(Connection connection, SegmentInfo addsi) throws SQLException, IOException { if (log.isDebugEnabled()) log.debug("Updating local segment from databse " + addsi); PreparedStatement segmentSelect = null; ResultSet resultSet = null; try { segmentSelect = connection.prepareStatement("select version_ from search_segments where name_ = ?"); segmentSelect.clearParameters(); segmentSelect.setString(1, addsi.getName()); resultSet = segmentSelect.executeQuery(); if (resultSet.next()) { InputStream packetStream = null; try { long version = resultSet.getLong(1); File f = new File(getSharedFileName(addsi.getName(), sharedStructuredStorage)); if (f.exists()) { packetStream = new FileInputStream(f); clusterStorage.unpackSegment(addsi, packetStream, version); if (log.isDebugEnabled()) log.debug("Updated Local " + addsi); } else { log.warn("Shared Segment file is missing " + f.getPath()); } } finally { try { packetStream.close(); } catch (Exception ex) { log.debug(ex); } } } else { log.error("Didnt find segment in database"); } } finally { try { resultSet.close(); } catch (Exception ex) { log.debug(ex); } try { segmentSelect.close(); } catch (Exception ex) { log.debug(ex); } } } public String getSharedSegments() { return sharedSegments; } public void setSharedSegments(String sharedSegments) { this.sharedSegments = sharedSegments; } public void dolog(String message) { if (debug) { log.info("JDBCClusterDebug :" + message); } else if (log.isDebugEnabled()) { log.debug("JDBCClusterDebug :" + message); } } public long getLastUpdate() { PreparedStatement segmentSelect = null; ResultSet resultSet = null; Connection connection = null; try { connection = dataSource.getConnection(); segmentSelect = connection .prepareStatement("select version_ from search_segments order by version_ desc"); segmentSelect.clearParameters(); resultSet = segmentSelect.executeQuery(); if (resultSet.next()) { return resultSet.getLong(1); } else { return 0; } } catch (Exception ex) { log.warn(" Cant find last update time " + ex.getClass().getName() + ":" + ex.getMessage()); return 0; } finally { try { resultSet.close(); } catch (Exception ex) { log.debug(ex); } try { segmentSelect.close(); } catch (Exception ex) { log.debug(ex); } try { connection.close(); } catch (Exception ex) { log.debug(ex); } } } public List<Object[]> getSegmentInfoList() { List<Object[]> seginfo = new ArrayList<Object[]>(); try { File searchDir = new File(searchIndexDirectory); long tsize = getSegmentInfoList(searchDir, seginfo); String size = null; if (tsize > 1024 * 1024 * 10) { size = String.valueOf(tsize / (1024 * 1024)) + "MB"; } else if (tsize >= 1024 * 1024) { size = String.valueOf(tsize / (1024 * 1024)) + "." + String.valueOf(tsize / (102 * 1024) + "MB"); } else { size = String.valueOf(tsize / (1024)) + "KB"; } seginfo.add(new Object[] { "Total", size, "" }); } catch (Exception ex) { seginfo.add(new Object[] { "Failed to get Segment Info list " + ex.getClass().getName() + " " + ex.getMessage() }); } return seginfo; } public long getSegmentInfoList(File searchDir, List<Object[]> seginfo) { File[] files = searchDir.listFiles(); long tsize = 0; if (files != null) { for (int i = 0; i < files.length; i++) { if (files[i].isDirectory()) { SegmentInfo sgi = SegmentInfoImpl.newLocalSegmentInfo(files[i], localStructuredStorage, searchIndexDirectory); if (sgi != null && sgi.isClusterSegment()) { String name = files[i].getName(); long lsize = sgi.getLocalSegmentSize(); tsize += lsize; long ts = sgi.getLocalSegmentLastModified(); String lastup = (new Date(ts)).toString(); String size = null; if (lsize > 1024 * 1024 * 10) { size = String.valueOf(lsize / (1024 * 1024)) + "MB"; } else if (lsize >= 1024 * 1024) { size = String.valueOf(lsize / (1024 * 1024)) + "." + String.valueOf(lsize / (102 * 1024) + "MB"); } else { size = String.valueOf(lsize / (1024)) + "KB"; } seginfo.add(new Object[] { name, size, lastup }); } else { tsize += getSegmentInfoList(files[i], seginfo); } } } } return tsize; } private void migrateSharedSegments() { if (localSegmentsOnly) { return; } if (sharedSegments != null && sharedSegments.length() > 0) { Connection connection = null; try { connection = dataSource.getConnection(); List<SegmentInfo> l = getDBSegments(connection); for (Iterator<SegmentInfo> li = l.iterator(); li.hasNext();) { SegmentInfo si = (SegmentInfo) li.next(); String shared = getSharedFileName(si.getName(), !sharedStructuredStorage); File f = new File(shared); if (f.exists()) { File fnew = new File(getSharedFileName(si.getName(), sharedStructuredStorage)); if (!fnew.getParentFile().mkdirs()) { log.warn("couldn't create directory: " + fnew.getParentFile().getPath()); } log.info("Moving " + f.getPath() + " to " + fnew.getPath()); if (!f.renameTo(fnew)) { log.warn("Failed rename " + f.getPath() + " to " + fnew.getPath()); } } } connection.commit(); } catch (Exception ex) { try { connection.rollback(); } catch (Exception ex1) { log.debug(ex); } } finally { try { connection.close(); } catch (Exception ex) { log.debug(ex); } } } } private void migrateLocalSegments() throws IOException { List<SegmentInfo> l = getLocalSegments(); for (Iterator<SegmentInfo> li = l.iterator(); li.hasNext();) { SegmentInfo si = li.next(); File f = SegmentInfoImpl.getSegmentLocation(si.getName(), !localStructuredStorage, searchIndexDirectory); if (f.exists()) { File fnew = SegmentInfoImpl.getSegmentLocation(si.getName(), localStructuredStorage, searchIndexDirectory); if (!fnew.getParentFile().mkdirs()) { log.warn("couldn't create directory " + fnew.getParentFile().getPath()); } log.info("Moving " + f.getPath() + " to " + fnew.getPath()); if (!f.renameTo(fnew)) { log.warn("Failed to rename " + f.getPath() + " to " + fnew.getPath()); } } } } public void getLock() throws IOException { if (parallelIndex) { throw new RuntimeException("Parallel index is not implemented yet"); } } public void releaseLock() { if (parallelIndex) { throw new RuntimeException("Parallel index is not implemented yet"); } } public boolean isMultipleIndexers() { return parallelIndex; } public boolean isParallelIndex() { return parallelIndex; } public void setParallelIndex(boolean parallelIndex) { this.parallelIndex = parallelIndex; } /** * @return Returns the localStructuredStorage. */ public boolean isLocalStructuredStorage() { return localStructuredStorage; } /** * @param localStructuredStorage * The localStructuredStorage to set. */ public void setLocalStructuredStorage(boolean localStructuredStorage) { this.localStructuredStorage = localStructuredStorage; } /** * @return Returns the sharedStructuredStorage. */ public boolean isSharedStructuredStorage() { return sharedStructuredStorage; } /** * @param sharedStructuredStorage * The sharedStructuredStorage to set. */ public void setSharedStructuredStorage(boolean sharedStructuredStorage) { this.sharedStructuredStorage = sharedStructuredStorage; } /** * @return the localSegmentsOnly */ public boolean isLocalSegmentsOnly() { return localSegmentsOnly; } /** * @param localSegmentsOnly * the localSegmentsOnly to set */ public void setLocalSegmentsOnly(boolean localSegmentsOnly) { this.localSegmentsOnly = localSegmentsOnly; } /** * @return the searchService */ public SearchService getSearchService() { return searchService; } /** * @param searchService * the searchService to set */ public void setSearchService(SearchService searchService) { this.searchService = searchService; } /* * (non-Javadoc) * * @see org.sakaiproject.search.index.ClusterFilesystem#centralIndexExists() */ public boolean centralIndexExists() { Connection connection = null; try { connection = dataSource.getConnection(); List<SegmentInfo> l = getDBSegments(connection); if (l != null && l.size() > 0) { return true; } return false; } catch (SQLException e) { return false; } finally { try { connection.close(); } catch (Exception ex) { log.debug(ex); } } } }