Java tutorial
/* * NOTE: This copyright does *not* cover user programs that use HQ * program services by normal system calls through the application * program interfaces provided as part of the Hyperic Plug-in Development * Kit or the Hyperic Client Development Kit - this is merely considered * normal use of the program, and does *not* fall under the heading of * "derived work". * * Copyright (C) [2004, 2005, 2006], Hyperic, Inc. * This file is part of HQ. * * HQ is free software; you can redistribute it and/or modify * it under the terms version 2 of the GNU General Public License as * published by the Free Software Foundation. This program is distributed * in the hope that it will be useful, but WITHOUT ANY WARRANTY; without * even the implied warranty of MERCHANTABILITY or FITNESS FOR A * PARTICULAR PURPOSE. See the GNU General Public License for more * details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 * USA. */ package org.hyperic.hq.agent.db; import java.io.BufferedInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.EOFException; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.RandomAccessFile; import java.util.ArrayList; import java.util.Collection; import java.util.ConcurrentModificationException; import java.util.Iterator; import java.util.NoSuchElementException; import java.util.Random; import java.util.SortedSet; import java.util.TreeSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.hyperic.hq.agent.stats.AgentStatsCollector; /** * A DiskList is a representation of a list on disk. The basic * usage is to add members to the end of the list, and remove * entries from it. The iterators returned support the remove() * operation, but will throw a ConcurrentModificationException * and fail-fast if an update is detected while iteration is occuring. * * The storage is contained in 2 files, one housing the data, and * the other housing the index. * * The format of the data file is as follows: * * [Record] * data - recordSize bytes containing the raw data * * The format of the index file is as follows: * * [Idx] * boolean - Indicates whether the record is in use or not * long int - Index of the previous record * long int - Index of the next record */ public class DiskList { private static final int IDX_REC_LEN = 1 + 8 + 8; private static final Log log = LogFactory.getLog(DiskList.class.getName()); private final String fileName; private final String idxFileName; private final RandomAccessFile indexFile; protected RandomAccessFile dataFile; private int recordSize; // Size of each record private long firstRec; // IDX of first record private long lastRec; // IDX of last record private final byte[] padBytes; // Utility array for padding protected SortedSet freeList; // Set(Long) of free rec idxs private int modNum; // Modification random number private final long checkSize; // Start to check for unused blocks // when the datafile reaches this // size in bytes private final int checkPerc; // Max percent (0-100) of free space // allowed in the data file. Only // significant when datafile size is // greated than checkSize private final long maxLength; // Max file size in bytes private final Random rand; private boolean closed; private static final AgentStatsCollector statsCollector = AgentStatsCollector.getInstance(); private static final String DISK_LIST_DISK_ITERATOR_REMOVE_TIME = AgentStatsCollector.DISK_LIST_DISK_ITERATOR_REMOVE_TIME; private static final String DISK_LIST_READ_RECORD_TIME = AgentStatsCollector.DISK_LIST_READ_RECORD_TIME; private static final String DISK_LIST_ADD_TO_LIST_TIME = AgentStatsCollector.DISK_LIST_ADD_TO_LIST_TIME; private static final String DISK_LIST_DELETE_ALL_RECORDS_TIME = AgentStatsCollector.DISK_LIST_DELETE_ALL_RECORDS_TIME; static { statsCollector.register(DISK_LIST_ADD_TO_LIST_TIME); statsCollector.register(DISK_LIST_READ_RECORD_TIME); statsCollector.register(DISK_LIST_DISK_ITERATOR_REMOVE_TIME); statsCollector.register(DISK_LIST_DELETE_ALL_RECORDS_TIME); } /** * Construct a new DiskList */ public DiskList(File dataFile, int recordSize, long checkSize, int checkPerc) throws IOException { this(dataFile, recordSize, checkSize, checkPerc, Long.MAX_VALUE); } /** * Construct a new DiskList * * @param dataFile the base location for the datafile. The index * file will be the same, with .idx appended * @param recordSize The maximum size for any record within the * data file. All records in the list will * occupy this many bytes worth of data * @param checkSize Size in to bytes to start checking for * unused blocks * @param checkPerc Maximum percentage of free blocks allowed when * data file is greater than checkSize. */ public DiskList(File dataFile, int recordSize, long checkSize, int checkPerc, long maxLength) throws IOException { File idxFile; idxFile = new File(dataFile + ".idx"); this.fileName = dataFile.getName(); this.idxFileName = idxFile.getName(); this.rand = new Random(); this.dataFile = new RandomAccessFile(dataFile, "rw"); this.recordSize = recordSize; this.padBytes = new byte[Math.max(recordSize, IDX_REC_LEN)]; this.modNum = this.rand.nextInt(); this.checkSize = checkSize; this.checkPerc = checkPerc; if (log.isDebugEnabled()) { log.debug("Setting max length for " + this.fileName + " to " + maxLength + " bytes"); } this.maxLength = maxLength; this.indexFile = new RandomAccessFile(idxFile, "rw"); this.genFreeList(idxFile); this.closed = false; } /** * Get the precentage of free space available in the datafile rounded * to the nearest whole number. (0-100) */ private long getDataFileFreePercentage() throws IOException { double dataBytes = this.dataFile.length(); double freeBytes = (this.freeList.size() * this.recordSize); return Math.round((freeBytes * 100) / dataBytes); } /** * Do maintenance on the data and index files. If the datafile size and * the free block percentage exceed the defined thresholds, the extra * free blocks will be removed by truncating the data and index files. * * Since truncation is used, some times it will be possible that even * though the criteria are met, we won't be able to delete the free space. * This is a recoverable situation though, since new blocks will be * inserted at the beginning of the data file. */ private void doMaintenence() throws IOException { long lastData = this.dataFile.length() / this.recordSize; long lastFree = ((Long) this.freeList.last()).longValue(); // Nothing we can do if the last block in the // file is not free if (lastData != (lastFree + 1)) { return; } // Simple iteration of the list. May be faster to do a // binary search using freeList.size() to determine if all // blocks are free, but this is more readable, and we are // dealing with small numbers (< a few million) long firstFree = lastFree; while (this.freeList.contains(new Long(firstFree - 1))) { firstFree--; } synchronized (this.dataFile) { // Truncate the data file this.dataFile.setLength(firstFree * this.recordSize); // Truncate the index file. this.indexFile.setLength(firstFree * IDX_REC_LEN); // Remove the free blocks deleted from the freelist. SortedSet subset = this.freeList.headSet(new Long(firstFree - 1)); // Must create a new TreeSet, since the sorted set imposes // restrictions on maximum and minimum key values. this.freeList = new TreeSet(subset); } long num = (lastFree - firstFree) + 1; this.log.info( "Deleted " + (num * this.recordSize) + " bytes from " + this.fileName + " (" + num + " blocks)"); } /** * A quick routine, which simply zips through the index file, * pulling out information about which records are free. * * We open up the file seperately here, so we can use the * buffered input stream, which makes our initial startup much * faster, if there is a lot of data sitting in the list. */ private void genFreeList(File idxFile) throws IOException { BufferedInputStream bIs; FileInputStream fIs = null; DataInputStream dIs; this.firstRec = -1; this.lastRec = -1; // TreeSet is used here to ensure a natural ordering of // the elements. this.freeList = new TreeSet(); try { fIs = new FileInputStream(idxFile); bIs = new BufferedInputStream(fIs); dIs = new DataInputStream(bIs); for (long idx = 0;; idx++) { boolean used; long prev, next; try { used = dIs.readBoolean(); } catch (EOFException exc) { break; } prev = dIs.readLong(); next = dIs.readLong(); if (used == false) { this.freeList.add(new Long(idx)); } else { if (prev == -1) { this.firstRec = idx; } if (next == -1) { this.lastRec = idx; } } } } catch (FileNotFoundException exc) { return; } finally { try { if (fIs != null) { fIs.close(); } } catch (IOException exc) { } } } /** * Add the string to the list of data being stored in the DiskList. * * @param data Data to add to the end of the list */ public void addToList(String data) throws IOException { if (this.closed) { throw new IOException("Datafile already closed"); } ByteArrayOutputStream bOs = new ByteArrayOutputStream(this.recordSize); DataOutputStream dOs = new DataOutputStream(bOs); dOs.writeUTF(data); if (bOs.size() > this.recordSize) { throw new IOException( "Data length(" + bOs.size() + ") exceeds " + "maximum record length(" + this.recordSize + ")"); } final long start = now(); bOs.write(this.padBytes, 0, this.recordSize - bOs.size()); byte[] bytes = bOs.toByteArray(); synchronized (this.dataFile) { Long firstFreeL; long firstFree; this.modNum = this.rand.nextInt(); try { firstFreeL = (Long) this.freeList.first(); firstFree = firstFreeL.longValue(); this.freeList.remove(firstFreeL); } catch (NoSuchElementException exc) { // Else we're adding to the end firstFree = this.indexFile.length() / IDX_REC_LEN; } // Write the record to the data file this.dataFile.seek(firstFree * this.recordSize); this.dataFile.write(bytes); bOs.reset(); dOs.writeBoolean(true); // Is Used dOs.writeLong(this.lastRec); // Previous record idx dOs.writeLong(-1); // Next record idx // Write the index for the record we just made this.indexFile.seek(firstFree * IDX_REC_LEN); bytes = bOs.toByteArray(); this.indexFile.write(bytes, 0, bytes.length); // Update the previous 'last' record to point to us if (this.lastRec != -1) { this.indexFile.seek((this.lastRec * IDX_REC_LEN) + 1 + 8); this.indexFile.writeLong(firstFree); } this.lastRec = firstFree; if (this.firstRec == -1) { this.firstRec = firstFree; } } if (this.dataFile.length() > this.maxLength) { this.log.error("Maximum file size for data file: " + this.fileName + " reached (" + this.maxLength + " bytes), truncating."); deleteAllRecords(); } long duration = now() - start; statsCollector.addStat(duration, DISK_LIST_ADD_TO_LIST_TIME); } private static long now() { return System.currentTimeMillis(); } private static class Record { private boolean isUsed; private long prevIdx; private long nextIdx; private String data; } private Record readRecord(long recNo) throws IOException { Record res = new Record(); if (recNo < 0) { throw new IllegalArgumentException("IDX must be positive"); } final long start = now(); synchronized (this.dataFile) { this.dataFile.seek(recNo * this.recordSize); res.data = this.dataFile.readUTF(); this.indexFile.seek(recNo * IDX_REC_LEN); res.isUsed = this.indexFile.readBoolean(); res.prevIdx = this.indexFile.readLong(); res.nextIdx = this.indexFile.readLong(); } final long duration = now() - start; statsCollector.addStat(duration, DISK_LIST_READ_RECORD_TIME); return res; } /** * Delete all the records from storage. */ public void deleteAllRecords() throws IOException { IOException sExc = null; if (this.closed) { throw new IOException("Datafile already closed"); } final long start = now(); synchronized (this.dataFile) { this.modNum = this.rand.nextInt(); this.firstRec = -1; this.lastRec = -1; try { this.indexFile.setLength(0); } catch (IOException exc) { this.log.error("IOException while truncating file " + idxFileName); if (this.log.isDebugEnabled()) { this.log.debug(exc); } sExc = exc; } try { this.dataFile.setLength(0); } catch (IOException exc) { this.log.error("IOException while truncating file " + fileName); if (this.log.isDebugEnabled()) { this.log.debug(exc); } if (sExc != null) { sExc = exc; } } this.freeList.clear(); } final long duration = now() - start; statsCollector.addStat(duration, DISK_LIST_DELETE_ALL_RECORDS_TIME); if (sExc != null) { throw sExc; } } public void removeRecord(long recNo) throws IOException { if (recNo < 0) { throw new IllegalArgumentException("IDX must be positive"); } synchronized (this.dataFile) { long prevIdx, nextIdx; this.modNum = this.rand.nextInt(); // Handle all the individual cases, to improve disk I/O performance // while maintaining data integrity if someone kills us during // the operation if (recNo == this.firstRec) { if (recNo == this.lastRec) { // It's the only record -- it's unused, and add to freeList this.firstRec = -1; this.lastRec = -1; } else { // It's the first in the list, but not the last this.indexFile.seek((recNo * IDX_REC_LEN) + 1 + 8); nextIdx = this.indexFile.readLong(); // Set next->prev to -1 this.indexFile.seek((nextIdx * IDX_REC_LEN) + 1); this.indexFile.writeLong(-1); this.firstRec = nextIdx; } } else if (recNo == this.lastRec) { // It's the last in the list, but not the first this.indexFile.seek((recNo * IDX_REC_LEN) + 1); prevIdx = this.indexFile.readLong(); // Set prev->next to -1 this.indexFile.seek((prevIdx * IDX_REC_LEN) + 1 + 8); this.indexFile.writeLong(-1); this.lastRec = prevIdx; } else { // Otherwise, it's somewhere in the middle, so we have to // update both the previous and next this.indexFile.seek((recNo * IDX_REC_LEN) + 1); prevIdx = this.indexFile.readLong(); nextIdx = this.indexFile.readLong(); // Set prev->next = next this.indexFile.seek((prevIdx * IDX_REC_LEN) + 1 + 8); this.indexFile.writeLong(nextIdx); // Set next->prev = prev this.indexFile.seek((nextIdx * IDX_REC_LEN) + 1); this.indexFile.writeLong(prevIdx); } this.indexFile.seek(recNo * IDX_REC_LEN); this.indexFile.writeBoolean(false); this.freeList.add(new Long(recNo)); } long length = this.dataFile.length(); long percFree = this.getDataFileFreePercentage(); if ((length > this.checkSize) && (percFree > this.checkPerc)) { this.doMaintenence(); } } /** * Close the DiskList. All subsequent methods will * result in an IOException being thrown. */ public void close() throws IOException { IOException sExc = null; if (this.closed) { throw new IOException("Datafile already closed"); } this.closed = true; try { this.dataFile.close(); } catch (IOException exc) { this.log.error("IOException while closing file " + fileName); if (this.log.isDebugEnabled()) { this.log.debug(exc); } sExc = exc; } try { this.indexFile.close(); } catch (IOException exc) { this.log.error("IOException while closing file " + idxFileName); if (this.log.isDebugEnabled()) { this.log.debug(exc); } if (sExc == null) { sExc = exc; } } if (sExc != null) { throw sExc; } } /** * This method converts lists from the old record size to the current one - * it reads all the records from the list using the old size, deletes the list * and than saves all the records using the current record size. * should be used when starting the first time after an upgrade. In version * 4.6.5 the default record size was changed from 1024 to 4000 and when we * will try to read the records with size 4000 we will get an exception because the * records size is 1024. This is a fix for Jira bug [HHQ-5387]. * @param oldSize - the old size of the record * @throws IOException */ public void convertListToCurrentRecordSize(int oldSize) throws IOException { log.info("Converting list on file '" + this.fileName + "' from size " + oldSize + " to size " + this.recordSize); int realRecSize = this.recordSize; this.recordSize = oldSize; Collection<String> records = new ArrayList<String>(); Iterator<String> iter = getListIterator(); for (; (iter != null) && iter.hasNext();) { String data = iter.next(); records.add(data); } log.info("Read " + records.size() + " records from file '" + this.fileName + "'"); deleteAllRecords(); this.recordSize = realRecSize; for (String rec : records) { addToList(rec); } } public static class DiskListIterator implements Iterator<String> { private final DiskList diskList; // Pointer back to the creating DiskList private long nextIdx; // Next index to read (or -1) private long curIdx; private boolean calledNext; private int modNum; private DiskListIterator(DiskList diskList, long nextIdx, int modNum) { this.diskList = diskList; this.nextIdx = nextIdx; this.curIdx = -1; this.calledNext = false; this.modNum = modNum; } public boolean hasNext() { return this.nextIdx != -1; } public String next() throws NoSuchElementException { Record rec; if (this.nextIdx == -1) { throw new NoSuchElementException(); } this.curIdx = this.nextIdx; synchronized (this.diskList.dataFile) { if (this.diskList.modNum != this.modNum) { throw new ConcurrentModificationException(); } try { rec = this.diskList.readRecord(this.curIdx); } catch (IOException e) { log.error("IOException while reading record"); if (log.isDebugEnabled()) { log.debug("IOException while trying to read record number " + this.curIdx, e); } NoSuchElementException ex = new NoSuchElementException("Error getting next element: " + e); ex.initCause(e); throw ex; } } this.nextIdx = rec.nextIdx; this.calledNext = true; return rec.data; } public void remove() { if (!this.calledNext) { throw new IllegalStateException("remove() called without first calling next()"); } this.calledNext = false; final long start = now(); synchronized (this.diskList.dataFile) { if (this.diskList.modNum != this.modNum) { throw new ConcurrentModificationException(); } try { this.diskList.removeRecord(this.curIdx); } catch (IOException exc) { log.error("IOException while removing record"); if (log.isDebugEnabled()) { log.debug(exc, exc); } throw new IllegalStateException("Error removing record: " + exc, exc); } this.modNum = this.diskList.modNum; } final long duration = now() - start; statsCollector.addStat(duration, DISK_LIST_DISK_ITERATOR_REMOVE_TIME); } } public Iterator<String> getListIterator() { synchronized (this.dataFile) { // XXX -- This is broken, and is used to satisfy a lame // requirement I made on the AgentStorageProvider interface.. :-( if (this.firstRec == -1) { if (log.isDebugEnabled()) { log.debug("getListIterator() - list '" + this.fileName + "' has no elements"); } return null; } return new DiskListIterator(this, this.firstRec, this.modNum); } } public static void main(String[] args) throws Exception { DiskList d; long NUM = 1024 * 128; long count; System.out.println("Creating DiskList.."); d = new DiskList(new File("mydb"), 1024, 2 * 1024 * 1024, 10); // Fill the entire file with data System.out.println("Adding " + NUM + " records.."); for (int i = 0; i < NUM; i++) { d.addToList("one " + i); } // Remove 1/2 of the data count = NUM / 2; System.out.println("Removing " + (NUM / 2) + " records.."); for (Iterator i = d.getListIterator(); i.hasNext() && (count > 0); count--) { String val = (String) i.next(); i.remove(); } // Add back 1/4 of the data System.out.println("Adding " + (NUM / 4) + " records.."); for (int i = 0; i < (NUM / 4); i++) { d.addToList("two " + i); } // Remove 1/2 of the data count = NUM / 2; System.out.println("Removing " + (NUM / 2) + " records.."); for (Iterator i = d.getListIterator(); i.hasNext() && (count > 0); count--) { String val = (String) i.next(); i.remove(); } // Add back 1/4 of the data System.out.println("Adding " + (NUM / 4) + " records.."); for (int i = 0; i < (NUM / 4); i++) { d.addToList("three " + i); } // Remove all data System.out.println("Removing all data.."); for (Iterator i = d.getListIterator(); i.hasNext();) { String val = (String) i.next(); i.remove(); } // Add back 1/16 of the data System.out.println("Adding " + (NUM / 16) + " records.."); for (int i = 0; i < (NUM / 4); i++) { d.addToList("three " + i); } // Remove all data System.out.println("Removing all data.."); for (Iterator i = d.getListIterator(); i.hasNext();) { String val = (String) i.next(); i.remove(); } d.close(); } }