Java tutorial
/********************************************************************************** * $URL: https://source.sakaiproject.org/svn/search/trunk/search-impl/impl/src/java/org/sakaiproject/search/index/impl/SegmentInfoImpl.java $ * $Id: SegmentInfoImpl.java 105078 2012-02-24 23:00:38Z ottenhoff@longsight.com $ *********************************************************************************** * * Copyright (c) 2003, 2004, 2005, 2006, 2007, 2008 The Sakai Foundation * * Licensed under the Educational Community License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.opensource.org/licenses/ECL-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * **********************************************************************************/ package org.sakaiproject.search.index.impl; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.nio.channels.FileLock; import java.util.Date; import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.sakaiproject.search.index.SegmentInfo; /** * @author ieb */ /** * segment info contains information on the segment, name, version, in db * * @author ieb */ public class SegmentInfoImpl implements SegmentInfo { private static final Log log = LogFactory.getLog(SegmentInfoImpl.class); public static final String TIMESTAMP_FILE = "_sakaicluster"; // if this file exists in the segment, this is a new segment that has not // been completed public static final String NEW_FILE = "_newsegment"; // if this file exists in the segment, the segment has been deleted public static final String DELETED_FILE = "_deletedsegment"; private String name; private long version; private boolean indb; private String[] states = new String[] { "new", "created", "deleted" }; private boolean localStructuredStorage; private String searchIndexDirectory; private File segmentLocation; private File timestampFile; private File newFile; private File deletedFile; private long segmentSize; public static final int STATE_NEW = 0; public static final int STATE_CREATED = 1; public static final int STATE_DELETED = 2; private SegmentState storedSegmentState = null; private SegmentState liveSegmentState = null; private static ConcurrentHashMap<String, String> lock = new ConcurrentHashMap<String, String>();; public String toString() { return name + ":" + version + ":" + indb + ": State:" + states[getState()] + ":Created:" + name + ": Update" + new Date(version); } public static SegmentInfo newSharedSegmentInfo(String name, long version, boolean localStructuredStorage, String searchIndexDirectory) { return new SegmentInfoImpl(name, version, true, localStructuredStorage, searchIndexDirectory); } public static SegmentInfo newLocalSegmentInfo(File file, boolean localStructuredStorage, String searchIndexDirectory) { return new SegmentInfoImpl(file, false, localStructuredStorage, searchIndexDirectory); } /** * Create a new copy of the segment refreshing from the local disk. * * @param recoverSegInfo * @return */ public static SegmentInfo newLocalSegmentInfo(SegmentInfo recoverSegInfo) { SegmentInfoImpl si = (SegmentInfoImpl) recoverSegInfo; File segmentLocation = getSegmentLocation(si.getName(), si.localStructuredStorage, si.searchIndexDirectory); return new SegmentInfoImpl(segmentLocation, false, si.localStructuredStorage, si.searchIndexDirectory); } /** * Create Segment Info and set the version number explicity, normally as a * result of the Segment vomming from the DB * * @param name * @param version * @param indb * @param localStructuredStorage * @param searchIndexDirectory */ private SegmentInfoImpl(String name, long version, boolean indb, boolean localStructuredStorage, String searchIndexDirectory) { this.searchIndexDirectory = searchIndexDirectory; this.localStructuredStorage = localStructuredStorage; segmentLocation = getSegmentLocation(name, localStructuredStorage, searchIndexDirectory); timestampFile = new File(segmentLocation, TIMESTAMP_FILE); newFile = new File(segmentLocation, NEW_FILE); deletedFile = new File(segmentLocation, DELETED_FILE); this.indb = indb; this.version = version; this.name = name; try { storedSegmentState = new SegmentState(this, timestampFile); } catch (IOException e) { log.debug("Cant Load Stored Segment State"); } try { liveSegmentState = new SegmentState(this, null); liveSegmentState.setTimeStamp(version); } catch (IOException e) { log.debug("Cant Load Live Segment State"); } } /** * Initialize a local segment * * @param file * @param indb * @param localStructuredStorage * @param searchIndexDirectory */ private SegmentInfoImpl(File file, boolean indb, boolean localStructuredStorage, String searchIndexDirectory) { this.searchIndexDirectory = searchIndexDirectory; this.localStructuredStorage = localStructuredStorage; segmentLocation = file; timestampFile = new File(segmentLocation, TIMESTAMP_FILE); newFile = new File(segmentLocation, NEW_FILE); deletedFile = new File(segmentLocation, DELETED_FILE); this.indb = indb; this.name = file.getName(); /* * The stored segment state is null, hence we have a potentially broken * segment. We should set the version to -1 and let it update from the * database if it can. */ this.version = -1; if (segmentLocation.isDirectory()) { try { storedSegmentState = new SegmentState(this, timestampFile); this.version = storedSegmentState.getTimeStamp(); } catch (IOException e) { log.info("Segment (" + name + "): Cant Load Stored Segment State"); } try { // this might cause a performance problem with a structured // index // since this is called hierachically and so may generate a // checksum // of the entire tree. Perhapse we should look for the existance // of the lucene segments file before performing this operation liveSegmentState = new SegmentState(this, null); } catch (IOException e) { log.info("Segment (" + name + "): Cant Load Live Segment State"); } } } public void setInDb(boolean b) { indb = b; } public String getName() { return name; } public boolean isInDb() { return indb; } public long getVersion() { return version; } public void setVersion(long l) { version = l; } public void setState(int state) { if (state == STATE_NEW) { if (!newFile.exists()) { try { newFile.getParentFile().mkdirs(); newFile.createNewFile(); } catch (IOException e) { log.error("Segment (" + name + "): Failed to create new segment marker at " + newFile); } } if (deletedFile.exists()) { if (!deletedFile.delete()) { log.warn("Can't delete file " + deletedFile.getPath()); } } } if (state == STATE_CREATED) { if (deletedFile.exists()) { if (!deletedFile.delete()) { log.warn("can't delete file " + deletedFile.getPath()); } } if (newFile.exists()) { if (!newFile.delete()) { log.warn("Can't delte file " + newFile.getPath()); } } } if (state == STATE_DELETED) { if (!deletedFile.exists()) { try { if (!deletedFile.getParentFile().mkdirs()) { log.warn("Unable to create directory" + deletedFile.getParentFile().getPath()); } if (!deletedFile.createNewFile()) { log.warn("can't create file: " + deletedFile.getPath()); } } catch (IOException e) { log.error("Segment (" + name + "): Failed to create deleted segment marker at " + deletedFile); } } if (newFile.exists()) { if (!newFile.delete()) { log.warn("can't delete file " + newFile.getPath()); } } } } /* * public static int getState(String name, boolean localStructuredStorage, * String searchIndexDirectory) { File f = getSegmentLocation(name, * localStructuredStorage, searchIndexDirectory); return getState(f); } */ public int getState() { if (deletedFile.exists()) { return STATE_DELETED; } else if (newFile.exists()) { return STATE_NEW; } else { return STATE_CREATED; } } public File getSegmentLocation() { return segmentLocation; } public static File getSegmentLocation(String name, boolean structured, String searchIndexDirectory) { if (structured) { String hashName = name.substring(name.length() - 4, name.length() - 2); File hash = new File(searchIndexDirectory, hashName); return new File(hash, name); } else { return new File(searchIndexDirectory, name); } } public void setNew() { setState(STATE_NEW); } public void setCreated() { setState(STATE_CREATED); } public void setDeleted() { setState(STATE_DELETED); } public boolean isNew() { return (getState() == STATE_NEW); } public boolean isCreated() { return (getState() == STATE_CREATED); } public boolean isDeleted() { return (getState() == STATE_DELETED); } /** * set the timestamp in the segment * * @param l * @throws IOException */ public void setTimeStamp(long l) throws IOException { liveSegmentState.analyze(this); liveSegmentState.setTimeStamp(l); liveSegmentState.save(timestampFile); if (!timestampFile.setLastModified(l)) { log.warn("Couldn't set file modification time on " + timestampFile.getPath()); } } public boolean isClusterSegment() { return (timestampFile.exists()); } /** * get the checksum out of the segment by the segment name * * @param segmentName * @return * @throws Exception * @throws IOException */ /* * private String getCheckSum(String segmentName) throws IOException { File * segmentdir = SegmentInfo.getSegmentLocation(segmentName, * localStructuredStorage, searchIndexDirectory); return * getCheckSum(segmentLocation); } */ public boolean checkSegmentValidity(boolean logging, String message) { /** * Dont check new segments, there will not be any state to check */ if (isCreated()) { return liveSegmentState.checkValidity(logging, message, storedSegmentState); } return true; } public long getLocalSegmentLastModified() { long lm = segmentLocation.lastModified(); if (segmentLocation.isDirectory()) { File[] l = segmentLocation.listFiles(); for (int i = 0; i < l.length; i++) { if (l[i].lastModified() > lm) { lm = l[i].lastModified(); } } } return lm; } public long getLocalSegmentSize() { if (segmentLocation.isDirectory()) { long lm = 0; File[] l = segmentLocation.listFiles(); for (int i = 0; i < l.length; i++) { lm += l[i].length(); } return lm; } else { return segmentLocation.length(); } } public long getTotalSize() { return getTotalSize(segmentLocation); } private long getTotalSize(File currentSegment) { long totalSize = 0; if (currentSegment.isDirectory()) { File[] files = currentSegment.listFiles(); if (files != null) { for (int i = 0; i < files.length; i++) { if (files[i].isDirectory()) { totalSize += getTotalSize(files[i]); } totalSize += files[i].length(); } } } else { totalSize += currentSegment.length(); } return totalSize; } public void touchSegment() throws IOException { setTimeStamp(System.currentTimeMillis()); } /* * (non-Javadoc) * * @see org.sakaiproject.search.index.SegmentInfo#getSize() */ public long getSize() { return segmentSize; } /* * (non-Javadoc) * * @see org.sakaiproject.search.index.SegmentInfo#loadSize() */ public void loadSize() { segmentSize = getTotalSize(); } /* * (non-Javadoc) * * @see org.sakaiproject.search.index.SegmentInfo#doFinalDelete() */ public void doFinalDelete() { if (isDeleted()) { File f = getSegmentLocation(); deleteAll(f); log.info("Segment (" + name + "): Deleted "); } else { log.error("Segment (" + name + "): Attempt to delete current Segment Data " + this); } } /** * delete all files under this file and including this file * * @param f */ public static void deleteAll(File f) { if (f.isDirectory()) { File[] files = f.listFiles(); if (files != null) { for (int i = 0; i < files.length; i++) { if (files[i].isDirectory()) { deleteAll(files[i]); } else { if (!files[i].delete()) { if (files[i].exists()) { log.warn("Failed to delete " + files[i].getPath()); } } } } } } if (!f.delete()) { if (f.exists()) { log.warn("Failed to delete " + f.getPath()); } } } /* * (non-Javadoc) * * @see org.sakaiproject.search.index.SegmentInfo#isLocalLock() */ public boolean isLocalLock() { String threadName = lock.get(this.name); if (threadName == null || threadName.equals(Thread.currentThread().getName())) { return true; } return false; } /* * (non-Javadoc) * * @see org.sakaiproject.search.index.SegmentInfo#lockLocalSegment() */ public void lockLocalSegment() { String threadName = lock.get(this.name); if (threadName == null) { lock.put(this.name, Thread.currentThread().getName()); } } /* * (non-Javadoc) * * @see org.sakaiproject.search.index.SegmentInfo#unlockLocalSegment() */ public void unlockLocalSegment() { if (isLocalLock()) { lock.remove(this.name); } } /* * (non-Javadoc) * * @see org.sakaiproject.search.index.SegmentInfo#debugSegment(java.lang.String) */ public void debugSegment(String message) { liveSegmentState.checkValidity(true, message, storedSegmentState); dumpAllFiles(getSegmentLocation(), message); } /** * @param message */ private void dumpAllFiles(File f, String message) { dumpFileInfo(f, message); File[] files = f.listFiles(); if (files != null) { for (int i = 0; i < files.length; i++) { if (files[i].isDirectory()) { dumpAllFiles(files[i], message); } else { dumpFileInfo(files[i], message); } } } } /** * @param f * @param message */ private void dumpFileInfo(File f, String message) { String fileInfo = message + "(" + f.getPath() + "):"; if (!f.exists()) { log.error(fileInfo + " File No longer exists"); } else { log.info(fileInfo + " size=[" + f.length() + "] lastModified=[" + f.lastModified() + "] read=[" + f.canRead() + "] write=[" + f.canWrite() + "] hidden=[" + f.isHidden() + "]"); try { FileInputStream fin = new FileInputStream(f); fin.read(new byte[4096]); log.info(fileInfo + " readOk"); FileLock fl = fin.getChannel().tryLock(); fl.release(); fin.close(); log.info(fileInfo + " lockOk"); } catch (Exception ex) { log.warn(fileInfo + " Lock or Read failed: ", ex); } } } /* * (non-Javadoc) * * @see org.sakaiproject.search.index.SegmentInfo#compareTo(java.lang.String, * org.sakaiproject.search.index.SegmentInfo) */ public void compareTo(String message, SegmentInfo compare) { SegmentInfoImpl si = (SegmentInfoImpl) compare; liveSegmentState.checkValidity(true, message, si.liveSegmentState); } }