Java tutorial
/* * #%L * Netarchivesuite - archive * %% * Copyright (C) 2005 - 2014 The Royal Danish Library, the Danish State and University Library, * the National Library of France and the Austrian National Library. * %% * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as * published by the Free Software Foundation, either version 2.1 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Lesser Public License for more details. * * You should have received a copy of the GNU General Lesser Public * License along with this program. If not, see * <http://www.gnu.org/licenses/lgpl-2.1.html>. * #L% */ package dk.netarkivet.archive.arcrepositoryadmin; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.sql.Connection; import java.sql.Date; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Random; import java.util.Set; import org.apache.commons.io.LineIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import dk.netarkivet.common.distribute.Channels; import dk.netarkivet.common.distribute.arcrepository.Replica; import dk.netarkivet.common.distribute.arcrepository.ReplicaStoreState; import dk.netarkivet.common.distribute.arcrepository.ReplicaType; import dk.netarkivet.common.exceptions.ArgumentNotValid; import dk.netarkivet.common.exceptions.IOFailure; import dk.netarkivet.common.exceptions.IllegalState; import dk.netarkivet.common.exceptions.UnknownID; import dk.netarkivet.common.utils.DBUtils; import dk.netarkivet.common.utils.ExceptionUtils; import dk.netarkivet.common.utils.FileUtils; import dk.netarkivet.common.utils.KeyValuePair; import dk.netarkivet.common.utils.NotificationType; import dk.netarkivet.common.utils.NotificationsFactory; import dk.netarkivet.common.utils.StringUtils; import dk.netarkivet.common.utils.TimeUtils; import dk.netarkivet.common.utils.batch.ChecksumJob; /** * Method for storing the bitpreservation cache in a database. * <p> * This method uses the 'admin.data' file for retrieving the upload status. */ public final class ReplicaCacheDatabase implements BitPreservationDAO { /** The log. */ protected static final Logger log = LoggerFactory.getLogger(ReplicaCacheDatabase.class); /** The current instance. */ private static ReplicaCacheDatabase instance; /** * The number of entries between logging in either file list or checksum list. This also controls how often the * database connection is renewed in methods {@link #addChecksumInformation(File, Replica)} and * {@link #addFileListInformation(File, Replica)}, where the operations can take hours, and seems to leak memory. */ private final int LOGGING_ENTRY_INTERVAL = 1000; /** Waiting time in seconds before attempting to initialise the database again. */ private final int WAIT_BEFORE_INIT_RETRY = 30; /** Number of DB INIT retries. */ private final int INIT_DB_RETRIES = 3; /** * Constructor. throws IllegalState if unable to initialize the database. */ private ReplicaCacheDatabase() { // Get a connection to the archive database Connection con = ArchiveDBConnection.get(); try { int retries = 0; boolean initialized = false; while (retries < INIT_DB_RETRIES && !initialized) { retries++; try { initialiseDB(con); initialized = true; log.info("Initialization of database successful"); return; } catch (IOFailure e) { if (retries < INIT_DB_RETRIES) { log.info( "Initialization failed. Probably because another application is calling the same " + "method now. Retrying after a minimum of {} seconds: ", WAIT_BEFORE_INIT_RETRY, e); waitSome(); } else { throw new IllegalState("Unable to initialize the database."); } } } } finally { ArchiveDBConnection.release(con); } } /** * Wait a while. */ private void waitSome() { Random rand = new Random(); try { Thread.sleep( WAIT_BEFORE_INIT_RETRY * TimeUtils.SECOND_IN_MILLIS + rand.nextInt(WAIT_BEFORE_INIT_RETRY)); } catch (InterruptedException e1) { // Ignored } } /** * Method for retrieving the current instance of this class. * * @return The current instance. */ public static synchronized ReplicaCacheDatabase getInstance() { if (instance == null) { instance = new ReplicaCacheDatabase(); } return instance; } /** * Method for initialising the database. This basically makes sure that all the replicas are within the database, * and that no unknown replicas have been defined. * * @param connection An open connection to the archive database */ protected void initialiseDB(Connection connection) { // retrieve the list of replicas. Collection<Replica> replicas = Replica.getKnown(); // Retrieve the replica IDs currently in the database. List<String> repIds = ReplicaCacheHelpers.retrieveIdsFromReplicaTable(connection); log.debug("IDs for replicas already in the database: {}", StringUtils.conjoin(",", repIds)); for (Replica rep : replicas) { // try removing the id from the temporary list of IDs within the DB. // If the remove is not successful, then the replica is already // in the database. if (!repIds.remove(rep.getId())) { // if the replica id cannot be removed from the list, then it // does not exist in the database and must be added. log.info("Inserting replica '{}' in database.", rep.toString()); ReplicaCacheHelpers.insertReplicaIntoDB(rep, connection); } else { // Otherwise it already exists in the DB. log.debug("Replica '{}' already inserted in database.", rep.toString()); } } // If unknown replica ids are found, then throw exception. if (repIds.size() > 0) { throw new IllegalState("The database contain identifiers for the following replicas, which are not " + "defined in the settings: " + repIds); } } /** * Method for retrieving the entry in the replicafileinfo table for a given file and replica. * * @param filename The name of the file for the entry. * @param replica The replica of the entry. * @return The replicafileinfo entry corresponding to the given filename and replica. * @throws ArgumentNotValid If the filename is either null or empty, or if the replica is null. */ public ReplicaFileInfo getReplicaFileInfo(String filename, Replica replica) throws ArgumentNotValid { ArgumentNotValid.checkNotNullOrEmpty(filename, "String filename"); ArgumentNotValid.checkNotNull(replica, "Replica replica"); // retrieve replicafileinfo for the given filename // FIXME Use joins! String sql = "SELECT replicafileinfo_guid, replica_id, " + "replicafileinfo.file_id, " + "segment_id, checksum, upload_status, filelist_status, " + "checksum_status, filelist_checkdatetime, checksum_checkdatetime " + "FROM replicafileinfo, file " + " WHERE file.file_id = replicafileinfo.file_id" + " AND file.filename=? AND replica_id=?"; PreparedStatement s = null; Connection con = ArchiveDBConnection.get(); try { s = DBUtils.prepareStatement(con, sql, filename, replica.getId()); ResultSet res = s.executeQuery(); if (res.next()) { // return the corresponding replica file info. return new ReplicaFileInfo(res); } else { return null; } } catch (SQLException e) { final String message = "SQL error while selecting ResultsSet by executing statement '" + sql + "'."; log.warn(message, e); throw new IOFailure(message, e); } finally { DBUtils.closeStatementIfOpen(s); ArchiveDBConnection.release(con); } } /** * Method for retrieving the checksum for a specific file. Since a file is not directly attached with a checksum, * the checksum of a file must be found by having the replicafileinfo entries for the file vote about it. * * @param filename The name of the file, whose checksum are to be found. * @return The checksum of the file, or a Null if no validated checksum can be found. * @throws ArgumentNotValid If the filename is either null or the empty string. */ public String getChecksum(String filename) throws ArgumentNotValid { ArgumentNotValid.checkNotNullOrEmpty(filename, "String filename"); Connection con = ArchiveDBConnection.get(); try { // retrieve the fileId long fileId = ReplicaCacheHelpers.retrieveIdForFile(filename, con); // Check if a checksum with status OK for the file can be found in // the database for (Replica rep : Replica.getKnown()) { // Return the checksum, if it has a valid status. if (ReplicaCacheHelpers.retrieveChecksumStatusForReplicaFileInfoEntry(fileId, rep.getId(), con) == ChecksumStatus.OK) { return ReplicaCacheHelpers.retrieveChecksumForReplicaFileInfoEntry(fileId, rep.getId(), con); } } // log that we vote about the file. log.debug("No commonly accepted checksum for the file '{}' has previously been found. " + "Voting to achieve one.", filename); // retrieves all the UNKNOWN_STATE checksums, and return if unanimous. Set<String> checksums = new HashSet<String>(); for (Replica rep : Replica.getKnown()) { if (ReplicaCacheHelpers.retrieveChecksumStatusForReplicaFileInfoEntry(fileId, rep.getId(), con) != ChecksumStatus.CORRUPT) { String tmpChecksum = ReplicaCacheHelpers.retrieveChecksumForReplicaFileInfoEntry(fileId, rep.getId(), con); if (tmpChecksum != null) { checksums.add(tmpChecksum); } else { log.info("Replica '{}' has a null checksum for the file '{}'.", rep.getId(), ReplicaCacheHelpers.retrieveFilenameForFileId(fileId, con)); } } } // check if unanimous (thus exactly one!) if (checksums.size() == 1) { // return the first and only value. return checksums.iterator().next(); } // If no checksums are found, then return null. if (checksums.size() == 0) { log.warn("No checksums found for file '{}'.", filename); return null; } log.info("No unanimous checksum found for file '{}'.", filename); // put all into a list for voting List<String> checksumList = new ArrayList<String>(); for (Replica rep : Replica.getKnown()) { String cs = ReplicaCacheHelpers.retrieveChecksumForReplicaFileInfoEntry(fileId, rep.getId(), con); if (cs != null) { checksumList.add(cs); } else { // log when it is second time we find this checksum to be null? log.debug("Replica '{}' has a null checksum for the file '{}'.", rep.getId(), ReplicaCacheHelpers.retrieveFilenameForFileId(fileId, con)); } } // vote and return the most frequent checksum. return ReplicaCacheHelpers.vote(checksumList); } finally { ArchiveDBConnection.release(con); } } /** * Retrieves the names of all the files in the file table of the database. * * @return The list of filenames known by the database. */ public Collection<String> retrieveAllFilenames() { Connection con = ArchiveDBConnection.get(); // make sql query. final String sql = "SELECT filename FROM file"; try { // Perform the select. return DBUtils.selectStringList(con, sql, new Object[] {}); } finally { ArchiveDBConnection.release(con); } } /** * Retrieves the ReplicaStoreState for the entry in the replicafileinfo table, which refers to the given file and * replica. * * @param filename The name of the file in the filetable. * @param replicaId The id of the replica. * @return The ReplicaStoreState for the specified entry. * @throws ArgumentNotValid If the replicaId or the filename are eihter null or the empty string. */ public ReplicaStoreState getReplicaStoreState(String filename, String replicaId) throws ArgumentNotValid { ArgumentNotValid.checkNotNullOrEmpty(filename, "String filename"); ArgumentNotValid.checkNotNullOrEmpty(replicaId, "String replicaId"); Connection con = ArchiveDBConnection.get(); // Make query for extracting the upload status. // FIXME Use joins. String sql = "SELECT upload_status FROM replicafileinfo, file WHERE " + "replicafileinfo.file_id = file.file_id AND file.filename = ? " + "AND replica_id = ?"; try { // execute the query. int ordinal = DBUtils.selectIntValue(con, sql, filename, replicaId); // return the corresponding ReplicaStoreState. return ReplicaStoreState.fromOrdinal(ordinal); } finally { ArchiveDBConnection.release(con); } } /** * Sets the ReplicaStoreState for the entry in the replicafileinfo table. * * @param filename The name of the file in the filetable. * @param replicaId The id of the replica. * @param state The ReplicaStoreState for the specified entry. * @throws ArgumentNotValid If the replicaId or the filename are eihter null or the empty string. Or if the * ReplicaStoreState is null. */ public void setReplicaStoreState(String filename, String replicaId, ReplicaStoreState state) throws ArgumentNotValid { ArgumentNotValid.checkNotNullOrEmpty(filename, "String filename"); ArgumentNotValid.checkNotNullOrEmpty(replicaId, "String replicaId"); ArgumentNotValid.checkNotNull(state, "ReplicaStoreState state"); Connection con = ArchiveDBConnection.get(); PreparedStatement statement = null; try { // retrieve the guid for the file. long fileId = ReplicaCacheHelpers.retrieveIdForFile(filename, con); // Make query for updating the upload status if (state == ReplicaStoreState.UPLOAD_COMPLETED) { // An UPLOAD_COMPLETE // UPLOAD_COMPLETE => filelist_status = OK, checksum_status = OK String sql = "UPDATE replicafileinfo SET upload_status = ?, " + "filelist_status = ?, checksum_status = ? " + "WHERE replica_id = ? AND file_id = ?"; statement = DBUtils.prepareStatement(con, sql, state.ordinal(), FileListStatus.OK.ordinal(), ChecksumStatus.OK.ordinal(), replicaId, fileId); } else { String sql = "UPDATE replicafileinfo SET upload_status = ? WHERE replica_id = ? AND file_id = ?"; statement = DBUtils.prepareStatement(con, sql, state.ordinal(), replicaId, fileId); } // execute the update and commit to database. statement.executeUpdate(); con.commit(); } catch (SQLException e) { String errMsg = "Received the following SQL error while updating the database: " + ExceptionUtils.getSQLExceptionCause(e); log.warn(errMsg, e); throw new IOFailure(errMsg, e); } finally { DBUtils.closeStatementIfOpen(statement); ArchiveDBConnection.release(con); } } /** * Creates a new entry for the filename for each replica, and give it the given checksum and set the upload_status = * UNKNOWN_UPLOAD_STATUS. * * @param filename The name of the file. * @param checksum The checksum of the file. * @throws ArgumentNotValid If the filename or the checksum is either null or the empty string. * @throws IllegalState If the file exists with another checksum on one of the replicas. Or if the file has already * been completely uploaded to one of the replicas. */ public void insertNewFileForUpload(String filename, String checksum) throws ArgumentNotValid, IllegalState { ArgumentNotValid.checkNotNullOrEmpty(filename, "String filename"); ArgumentNotValid.checkNotNullOrEmpty(checksum, "String checkums"); Connection con = ArchiveDBConnection.get(); // retrieve the fileId for the filename. long fileId; try { // insert into DB, or make sure that it can be inserted. if (existsFileInDB(filename)) { // retrieve the fileId of the existing file. fileId = ReplicaCacheHelpers.retrieveIdForFile(filename, con); // Check the entries for this file associated with the replicas. for (Replica rep : Replica.getKnown()) { // Ensure that the file has not been completely uploaded to a // replica. ReplicaStoreState us = ReplicaCacheHelpers.retrieveUploadStatus(fileId, rep.getId(), con); if (us.equals(ReplicaStoreState.UPLOAD_COMPLETED)) { throw new IllegalState( "The file has already been completely uploaded to the replica: " + rep); } // make sure that it has not been attempted uploaded with // another checksum String entryCs = ReplicaCacheHelpers.retrieveChecksumForReplicaFileInfoEntry(fileId, rep.getId(), con); // throw an exception if the registered checksum differs. if (entryCs != null && !checksum.equals(entryCs)) { throw new IllegalState("The file '" + filename + "' with checksum '" + entryCs + "' has attempted being uploaded with the checksum '" + checksum + "'"); } } } else { fileId = ReplicaCacheHelpers.insertFileIntoDB(filename, con); } for (Replica rep : Replica.getKnown()) { // retrieve the guid for the corresponding replicafileinfo entry long guid = ReplicaCacheHelpers.retrieveReplicaFileInfoGuid(fileId, rep.getId(), con); // Update with the correct information. ReplicaCacheHelpers.updateReplicaFileInfo(guid, checksum, ReplicaStoreState.UNKNOWN_UPLOAD_STATE, con); } } finally { ArchiveDBConnection.release(con); } } /** * Method for inserting an entry into the database about a file upload has begun for a specific replica. It is not * tested whether the entry has another checksum or another UploadStatus. * * @param filename The name of the file. * @param replica The replica for the replicafileinfo. * @param state The new ReplicaStoreState for the entry. * @throws ArgumentNotValid If the filename is either null or the empty string. Or if the replica or the status is * null. */ public void changeStateOfReplicafileinfo(String filename, Replica replica, ReplicaStoreState state) throws ArgumentNotValid { ArgumentNotValid.checkNotNullOrEmpty(filename, "String filename"); ArgumentNotValid.checkNotNull(replica, "Replica rep"); ArgumentNotValid.checkNotNull(state, "ReplicaStoreState state"); PreparedStatement statement = null; Connection connection = null; try { connection = ArchiveDBConnection.get(); // retrieve the replicafileinfo_guid for this filename . long guid = ReplicaCacheHelpers.retrieveGuidForFilenameOnReplica(filename, replica.getId(), connection); statement = connection.prepareStatement( "UPDATE replicafileinfo SET upload_status = ? " + "WHERE replicafileinfo_guid = ?"); statement.setLong(1, state.ordinal()); statement.setLong(2, guid); // Perform the update. statement.executeUpdate(); connection.commit(); } catch (SQLException e) { throw new IllegalState("Cannot update status and checksum of a replicafileinfo in the database.", e); } finally { DBUtils.closeStatementIfOpen(statement); if (connection != null) { ArchiveDBConnection.release(connection); } } } /** * Method for inserting an entry into the database about a file upload has begun for a specific replica. It is not * tested whether the entry has another checksum or another UploadStatus. * * @param filename The name of the file. * @param checksum The new checksum for the entry. * @param replica The replica for the replicafileinfo. * @param state The new ReplicaStoreState for the entry. * @throws ArgumentNotValid If the filename or the checksum is either null or the empty string. Or if the replica or * the status is null. * @throws IllegalState If an sql exception is thrown. */ public void changeStateOfReplicafileinfo(String filename, String checksum, Replica replica, ReplicaStoreState state) throws ArgumentNotValid, IllegalState { ArgumentNotValid.checkNotNullOrEmpty(filename, "String filename"); ArgumentNotValid.checkNotNullOrEmpty(checksum, "String checksum"); ArgumentNotValid.checkNotNull(replica, "Replica rep"); ArgumentNotValid.checkNotNull(state, "ReplicaStoreState state"); PreparedStatement statement = null; Connection connection = null; try { connection = ArchiveDBConnection.get(); // retrieve the replicafileinfo_guid for this filename . long guid = ReplicaCacheHelpers.retrieveGuidForFilenameOnReplica(filename, replica.getId(), connection); statement = connection.prepareStatement("UPDATE replicafileinfo SET upload_status = ?, checksum = ? " + "WHERE replicafileinfo_guid = ?"); statement.setLong(1, state.ordinal()); statement.setString(2, checksum); statement.setLong(3, guid); // Perform the update. statement.executeUpdate(); connection.commit(); } catch (SQLException e) { throw new IllegalState("Cannot update status and checksum of a replicafileinfo in the database.", e); } finally { DBUtils.closeStatementIfOpen(statement); ArchiveDBConnection.release(connection); } } /** * Retrieves the names of all the files in the given replica which has the specified UploadStatus. * * @param replicaId The id of the replica which contain the files. * @param state The ReplicaStoreState for the wanted files. * @return The list of filenames for the entries in the replica which has the specified UploadStatus. * @throws ArgumentNotValid If the UploadStatus is null or if the replicaId is either null or the empty string. */ public Collection<String> retrieveFilenamesForReplicaEntries(String replicaId, ReplicaStoreState state) throws ArgumentNotValid { ArgumentNotValid.checkNotNull(state, "ReplicaStoreState state"); ArgumentNotValid.checkNotNullOrEmpty(replicaId, "String replicaId"); Connection con = ArchiveDBConnection.get(); final String sql = "SELECT filename FROM replicafileinfo " + "LEFT OUTER JOIN file ON replicafileinfo.file_id = file.file_id " + "WHERE replica_id = ? AND upload_status = ?"; try { return DBUtils.selectStringList(con, sql, replicaId, state.ordinal()); } finally { ArchiveDBConnection.release(con); } } /** * Checks whether a file is already in the file table in the database. * * @param filename The name of the file in the database. * @return Whether the file was found in the database. * @throws IllegalState If more than one entry with the given filename was found. */ public boolean existsFileInDB(String filename) throws IllegalState { // retrieve the amount of times this replica is within the database. Connection con = ArchiveDBConnection.get(); final String sql = "SELECT COUNT(*) FROM file WHERE filename = ?"; try { int count = DBUtils.selectIntValue(con, sql, filename); // Handle the different cases for count. switch (count) { case 0: return false; case 1: return true; default: throw new IllegalState("Cannot handle " + count + " files " + "with the name '" + filename + "'."); } } finally { ArchiveDBConnection.release(con); } } /** * Method for retrieving the filelist_status for a replicafileinfo entry. * * @param filename The name of the file. * @param replica The replica where the file should be. * @return The filelist_status for the file in the replica. * @throws ArgumentNotValid If the replica is null or the filename is either null or the empty string. */ public FileListStatus retrieveFileListStatus(String filename, Replica replica) throws ArgumentNotValid { ArgumentNotValid.checkNotNullOrEmpty(filename, "String filename"); ArgumentNotValid.checkNotNull(replica, "Replica replica"); Connection con = ArchiveDBConnection.get(); try { // retrieve the filelist_status for the entry. int status = ReplicaCacheHelpers.retrieveFileListStatusFromReplicaFileInfo(filename, replica.getId(), con); // Return the corresponding FileListStatus return FileListStatus.fromOrdinal(status); } finally { ArchiveDBConnection.release(con); } } /** * SQL used to update the checksum status of straightforward cases. See complete description for method below. */ public static final String updateChecksumStatusSql = "" + "UPDATE replicafileinfo SET checksum_status = " + ChecksumStatus.OK.ordinal() + " " + "WHERE checksum_status != " + ChecksumStatus.OK.ordinal() + " AND file_id IN ( " + " SELECT file_id " + " FROM ( " + " SELECT file_id, COUNT(file_id) AS checksums, SUM(replicas) replicas " + " FROM ( " + " SELECT file_id, COUNT(checksum) AS replicas, checksum " + " FROM replicafileinfo " + " WHERE filelist_status != " + FileListStatus.MISSING.ordinal() + " AND checksum IS NOT NULL " + " GROUP BY file_id, checksum " + " ) AS ss1 " + " GROUP BY file_id " + " ) AS ss2 " + " WHERE checksums = 1 " + ")"; /** * SQL used to select those files whose check status has to be voted on. See complete description for method below. */ public static final String selectForFileChecksumVotingSql = "" + "SELECT file_id " + "FROM ( " + " SELECT file_id, COUNT(file_id) AS checksums, SUM(replicas) replicas " + " FROM ( " + " SELECT file_id, COUNT(checksum) AS replicas, checksum " + " FROM replicafileinfo " + " WHERE filelist_status != " + FileListStatus.MISSING.ordinal() + " AND checksum IS NOT NULL " + " GROUP BY file_id, checksum " + " ) AS ss1 " + " GROUP BY file_id " + ") AS ss2 " + "WHERE checksums > 1 "; /** * This method is used to update the status for the checksums for all replicafileinfo entries. <br/> * <br/> * For each file in the database, the checksum vote is made in the following way. <br/> * Each entry in the replicafileinfo table containing the file is retrieved. All the unique checksums are retrieved, * e.g. if a checksum is found more than one, then it is ignored. <br/> * If only one unique checksum is found, then if must be the correct one, and all the replicas with this file will * have their checksum_status set to 'OK'. <br/> * If more than one checksum is found, then a vote for the correct checksum is performed. This is done by counting * the amount of time each of the unique checksum is found among the replicafileinfo entries for the current file. * The checksum with most votes is chosen as the correct one, and the checksum_status for all the replicafileinfo * entries with this checksum is set to 'OK', whereas the replicafileinfo entries with a different checksum is set * to 'CORRUPT'. <br/> * If no winner is found then a warning and a notification is issued, and the checksum_status for all the * replicafileinfo entries with for the current file is set to 'UNKNOWN'. <br/> */ public void updateChecksumStatus() { log.info("UpdateChecksumStatus operation commencing"); Connection con = ArchiveDBConnection.get(); boolean autoCommit = true; try { autoCommit = con.getAutoCommit(); // Set checksum_status to 'OK' where there is the same // checksum across all replicas. DBUtils.executeSQL(con, updateChecksumStatusSql); // Get all the fileids that need processing. // Previously: "SELECT file_id FROM file" Iterator<Long> fileIdsIterator = DBUtils.selectLongIterator(con, selectForFileChecksumVotingSql); // For each fileid while (fileIdsIterator.hasNext()) { long fileId = fileIdsIterator.next(); ReplicaCacheHelpers.fileChecksumVote(fileId, con); } } catch (SQLException e) { throw new IOFailure("Error getting auto commit.\n" + ExceptionUtils.getSQLExceptionCause(e), e); } finally { try { con.setAutoCommit(autoCommit); } catch (SQLException e) { log.error("Could not change auto commit back to default!"); } ArchiveDBConnection.release(con); } log.info("UpdateChecksumStatus operation completed!"); } /** * Method for updating the status for a specific file for all the replicas. If the checksums for the replicas differ * for some replica, then based on a checksum vote, a specific checksum is chosen as the 'correct' one, and the * entries with another checksum than the 'correct one' will be marked as corrupt. * * @param filename The name of the file to update the status for. * @throws ArgumentNotValid If the filename is either null or the empty string. */ @Override public void updateChecksumStatus(String filename) throws ArgumentNotValid { ArgumentNotValid.checkNotNullOrEmpty(filename, "String filename"); Connection con = ArchiveDBConnection.get(); try { // retrieve the id and vote! Long fileId = ReplicaCacheHelpers.retrieveIdForFile(filename, con); ReplicaCacheHelpers.fileChecksumVote(fileId, con); } finally { ArchiveDBConnection.release(con); } } /** * Given the output of a checksum job, add the results to the database. * <p> * The following fields in the table are updated for each corresponding entry in the replicafileinfo table: <br/> * - checksum = the given checksum. <br/> * - filelist_status = ok. <br/> * - filelist_checkdatetime = now. <br/> * - checksum_checkdatetime = now. * * @param checksumOutputFile The output of a checksum job in a file * @param replica The replica this checksum job is for. */ @Override public void addChecksumInformation(File checksumOutputFile, Replica replica) { // validate arguments ArgumentNotValid.checkNotNull(checksumOutputFile, "File checksumOutputFile"); ArgumentNotValid.checkNotNull(replica, "Replica replica"); // Sort the checksumOutputFile file. File sortedResult = new File(checksumOutputFile.getParent(), checksumOutputFile.getName() + ".sorted"); FileUtils.sortFile(checksumOutputFile, sortedResult); final long datasize = FileUtils.countLines(sortedResult); Set<Long> missingReplicaRFIs = null; Connection con = ArchiveDBConnection.get(); LineIterator lineIterator = null; try { // Make sure, that the replica exists in the database. if (!ReplicaCacheHelpers.existsReplicaInDB(replica, con)) { String msg = "Cannot add checksum information, since the replica '" + replica.toString() + "' does not exist within the database."; log.warn(msg); throw new IOFailure(msg); } log.info("Starting processing of {} checksum entries for replica {}", datasize, replica.getId()); // retrieve the list of files already known by this cache. // TODO This does not scale! Should the datastructure // (missingReplicaRFIs) be disk-bound in some way, or optimized // in some way, e.g. using it.unimi.dsi.fastutil.longs.LongArrayList missingReplicaRFIs = ReplicaCacheHelpers.retrieveReplicaFileInfoGuidsForReplica(replica.getId(), con); // Initialize the String iterator lineIterator = new LineIterator(new FileReader(sortedResult)); String lastFilename = ""; String lastChecksum = ""; int i = 0; while (lineIterator.hasNext()) { String line = lineIterator.next(); // log that it is in progress every so often. if ((i % LOGGING_ENTRY_INTERVAL) == 0) { log.info("Processed checksum list entry number {} for replica {}", i, replica); // Close connection, and open another one // to avoid memory-leak (NAS-2003) ArchiveDBConnection.release(con); con = ArchiveDBConnection.get(); log.debug("Databaseconnection has now been renewed"); } ++i; // parse the input. final KeyValuePair<String, String> entry = ChecksumJob.parseLine(line); final String filename = entry.getKey(); final String checksum = entry.getValue(); // check for duplicates if (filename.equals(lastFilename)) { // if different checksums, then if (!checksum.equals(lastChecksum)) { // log and send notification String errMsg = "Unidentical duplicates of file '" + filename + "' with the checksums '" + lastChecksum + "' and '" + checksum + "'. First instance used."; log.warn(errMsg); NotificationsFactory.getInstance().notify(errMsg, NotificationType.WARNING); } else { // log about duplicate identical log.debug("Duplicates of the file '{}' found with the same checksum '{}'.", filename, checksum); } // avoid overhead of inserting duplicates twice. continue; } // set these value to be the old values in next iteration. lastFilename = filename; lastChecksum = checksum; // Process the current (filename + checksum) combo for this replica // Remove the returned replicafileinfo guid from the missing entries. missingReplicaRFIs .remove(ReplicaCacheHelpers.processChecksumline(filename, checksum, replica, con)); } } catch (IOException e) { throw new IOFailure("Unable to read checksum entries from file", e); } finally { ArchiveDBConnection.release(con); LineIterator.closeQuietly(lineIterator); } con = ArchiveDBConnection.get(); try { // go through the not found replicafileinfo for this replica to change // their filelist_status to missing. if (missingReplicaRFIs.size() > 0) { log.warn("Found {} missing files for replica '{}'.", missingReplicaRFIs.size(), replica); for (long rfi : missingReplicaRFIs) { // set the replicafileinfo in the database to missing. ReplicaCacheHelpers.updateReplicaFileInfoMissingFromFilelist(rfi, con); } } // update the checksum updated date for this replica. ReplicaCacheHelpers.updateChecksumDateForReplica(replica, con); ReplicaCacheHelpers.updateFilelistDateForReplica(replica, con); log.info("Finished processing of {} checksum entries for replica {}", datasize, replica.getId()); } finally { ArchiveDBConnection.release(con); } } /** * Method for adding the results from a list of filenames on a replica. This list of filenames should return the * list of all the files within the database. * <p> * For each file in the FileListJob the following fields are set for the corresponding entry in the replicafileinfo * table: <br/> * - filelist_status = ok. <br/> * - filelist_checkdatetime = now. * <p> * For each entry in the replicafileinfo table for the replica which are missing in the results from the FileListJob * the following fields are assigned the following values: <br/> * - filelist_status = missing. <br/> * - filelist_checkdatetime = now. * * @param filelistFile The list of filenames either parsed from a FilelistJob or the result from a * GetAllFilenamesMessage. * @param replica The replica, which the FilelistBatchjob has run upon. * @throws ArgumentNotValid If the filelist or the replica is null. * @throws UnknownID If the replica does not already exist in the database. */ @Override public void addFileListInformation(File filelistFile, Replica replica) throws ArgumentNotValid, UnknownID { ArgumentNotValid.checkNotNull(filelistFile, "File filelistFile"); ArgumentNotValid.checkNotNull(replica, "Replica replica"); // Sort the filelist file. File sortedResult = new File(filelistFile.getParent(), filelistFile.getName() + ".sorted"); FileUtils.sortFile(filelistFile, sortedResult); final long datasize = FileUtils.countLines(sortedResult); Connection con = ArchiveDBConnection.get(); Set<Long> missingReplicaRFIs = null; LineIterator lineIterator = null; try { // Make sure, that the replica exists in the database. if (!ReplicaCacheHelpers.existsReplicaInDB(replica, con)) { String errorMsg = "Cannot add filelist information, since the replica '" + replica.toString() + "' does not exist in the database."; log.warn(errorMsg); throw new UnknownID(errorMsg); } log.info("Starting processing of {} filelist entries for replica {}", datasize, replica.getId()); // retrieve the list of files already known by this cache. // TODO This does not scale! Should this datastructure // (missingReplicaRFIs) be disk-bound in some way. missingReplicaRFIs = ReplicaCacheHelpers.retrieveReplicaFileInfoGuidsForReplica(replica.getId(), con); // Initialize String iterator lineIterator = new LineIterator(new FileReader(sortedResult)); String lastFileName = ""; int i = 0; while (lineIterator.hasNext()) { String file = lineIterator.next(); // log that it is in progress every so often. if ((i % LOGGING_ENTRY_INTERVAL) == 0) { log.info("Processed file list entry number {} for replica {}", i, replica); // Close connection, and open another one // to avoid memory-leak (NAS-2003) ArchiveDBConnection.release(con); con = ArchiveDBConnection.get(); log.debug("Databaseconnection has now been renewed"); } ++i; // handle duplicates. if (file.equals(lastFileName)) { log.warn("There have been found multiple files with the name '{}'", file); continue; } lastFileName = file; // Add information for one file, and remove the ReplicaRFI from the // set of missing ones. missingReplicaRFIs.remove(ReplicaCacheHelpers.addFileInformation(file, replica, con)); } } catch (IOException e) { throw new IOFailure("Unable to read the filenames from file", e); } finally { ArchiveDBConnection.release(con); LineIterator.closeQuietly(lineIterator); } con = ArchiveDBConnection.get(); try { // go through the not found replicafileinfo for this replica to change // their filelist_status to missing. if (missingReplicaRFIs.size() > 0) { log.warn("Found {} missing files for replica '{}'.", missingReplicaRFIs.size(), replica); for (long rfi : missingReplicaRFIs) { // set the replicafileinfo in the database to missing. ReplicaCacheHelpers.updateReplicaFileInfoMissingFromFilelist(rfi, con); } } // Update the date for filelist update for this replica. ReplicaCacheHelpers.updateFilelistDateForReplica(replica, con); } finally { ArchiveDBConnection.release(con); } } /** * Get the date for the last file list job. * * @param replica The replica to get the date for. * @return The date of the last missing files update for the replica. A null is returned if no last missing files * update has been performed. * @throws ArgumentNotValid If the replica is null. * @throws IllegalArgumentException If the Date of the Timestamp cannot be instantiated. */ @Override public Date getDateOfLastMissingFilesUpdate(Replica replica) throws ArgumentNotValid, IllegalArgumentException { ArgumentNotValid.checkNotNull(replica, "Replica replica"); Connection con = ArchiveDBConnection.get(); String result = null; try { // sql for retrieving this replicafileinfo_guid. String sql = "SELECT filelist_updated FROM replica WHERE replica_id = ?"; result = DBUtils.selectStringValue(con, sql, replica.getId()); } finally { ArchiveDBConnection.release(con); } // return null if the field has no be set for this replica. if (result == null) { log.debug( "The 'filelist_updated' field has not been set, as no missing files update has been performed yet."); return null; } else { // Parse the timestamp into a date. return new Date(Timestamp.valueOf(result).getTime()); } } /** * Method for retrieving the date for the last update for corrupted files. * <p> * This method does not contact the replicas, it only retrieves the data from the last time the checksum was * retrieved. * * @param replica The replica to find the date for the latest update for corruption of files. * @return The date for the last checksum update. A null is returned if no wrong files update has been performed for * this replica. * @throws ArgumentNotValid If the replica is null. * @throws IllegalArgumentException If the Date of the Timestamp cannot be instantiated. */ @Override public Date getDateOfLastWrongFilesUpdate(Replica replica) throws ArgumentNotValid, IllegalArgumentException { ArgumentNotValid.checkNotNull(replica, "Replica replica"); Connection con = ArchiveDBConnection.get(); String result = null; try { // The SQL statement for retrieving the date for last update of // checksum for the replica. final String sql = "SELECT checksum_updated FROM replica WHERE replica_id = ?"; result = DBUtils.selectStringValue(con, sql, replica.getId()); } finally { ArchiveDBConnection.release(con); } // return null if the field has no be set for this replica. if (result == null) { log.debug( "The 'checksum_updated' field has not been set, as no wrong files update has been performed yet."); return null; } else { // Parse the timestamp into a date. return new Date(Timestamp.valueOf(result).getTime()); } } /** * Method for retrieving the number of files missing from a specific replica. * <p> * This method does not contact the replica directly, it only retrieves the count of missing files from the last * filelist update. * * @param replica The replica to find the number of missing files for. * @return The number of missing files for the replica. * @throws ArgumentNotValid If the replica is null. */ @Override public long getNumberOfMissingFilesInLastUpdate(Replica replica) throws ArgumentNotValid { ArgumentNotValid.checkNotNull(replica, "Replica replica"); Connection con = ArchiveDBConnection.get(); // The SQL statement to retrieve the number of entries in the // replicafileinfo table with file_status set to either missing or // no_status for the replica. // FIXME Consider using a UNION instead of OR. final String sql = "SELECT COUNT(*) FROM replicafileinfo " + "WHERE replica_id = ? AND ( filelist_status = ? OR filelist_status = ?)"; try { return DBUtils.selectLongValue(con, sql, replica.getId(), FileListStatus.MISSING.ordinal(), FileListStatus.NO_FILELIST_STATUS.ordinal()); } finally { ArchiveDBConnection.release(con); } } /** * Method for retrieving the list of the names of the files which was missing for the replica in the last filelist * update. * <p> * This method does not contact the replica, it only uses the database to find the files, which was missing during * the last filelist update. * * @param replica The replica to find the list of missing files for. * @return A list containing the names of the files which are missing in the given replica. * @throws ArgumentNotValid If the replica is null. */ @Override public Iterable<String> getMissingFilesInLastUpdate(Replica replica) throws ArgumentNotValid { ArgumentNotValid.checkNotNull(replica, "Replica replica"); Connection con = ArchiveDBConnection.get(); // The SQL statement to retrieve the filenames of the missing // replicafileinfo to the given replica. final String sql = "SELECT filename FROM replicafileinfo " + "LEFT OUTER JOIN file ON replicafileinfo.file_id = file.file_id " + "WHERE replica_id = ? AND ( filelist_status = ? OR filelist_status = ? )"; try { return DBUtils.selectStringList(con, sql, replica.getId(), FileListStatus.MISSING.ordinal(), FileListStatus.NO_FILELIST_STATUS.ordinal()); } finally { ArchiveDBConnection.release(con); } } /** * Method for retrieving the amount of files with a incorrect checksum within a replica. * <p> * This method does not contact the replica, it only uses the database to count the amount of files which are * corrupt. * * @param replica The replica to find the number of corrupted files for. * @return The number of corrupted files. * @throws ArgumentNotValid If the replica is null. */ @Override public long getNumberOfWrongFilesInLastUpdate(Replica replica) throws ArgumentNotValid { ArgumentNotValid.checkNotNull(replica, "Replica"); Connection con = ArchiveDBConnection.get(); // The SQL statement to retrieve the number of corrupted entries in // the replicafileinfo table for the given replica. final String sql = "SELECT COUNT(*) FROM replicafileinfo WHERE replica_id = ? AND checksum_status = ?"; try { return DBUtils.selectLongValue(con, sql, replica.getId(), ChecksumStatus.CORRUPT.ordinal()); } finally { ArchiveDBConnection.release(con); } } /** * Method for retrieving the list of the files in the replica which have a incorrect checksum. E.g. the * checksum_status is set to CORRUPT. * <p> * This method does not contact the replica, it only uses the local database. * * @param replica The replica to find the list of corrupted files for. * @return The list of files which have wrong checksums. * @throws ArgumentNotValid If the replica is null. */ @Override public Iterable<String> getWrongFilesInLastUpdate(Replica replica) throws ArgumentNotValid { ArgumentNotValid.checkNotNull(replica, "Replica replica"); Connection con = ArchiveDBConnection.get(); // The SQL statement to retrieve the filenames for the corrupted files // in the replicafileinfo table for the given replica. String sql = "SELECT filename FROM replicafileinfo " + "LEFT OUTER JOIN file ON replicafileinfo.file_id = file.file_id " + "WHERE replica_id = ? AND checksum_status = ?"; try { return DBUtils.selectStringList(con, sql, replica.getId(), ChecksumStatus.CORRUPT.ordinal()); } finally { ArchiveDBConnection.release(con); } } /** * Method for retrieving the number of files within a replica. This count all the files which are not missing from * the replica, thus all entries in the replicafileinfo table which has the filelist_status set to OK. It is ignored * whether the files has a correct checksum. * <p> * This method does not contact the replica, it only uses the local database. * * @param replica The replica to count the number of files for. * @return The number of files within the replica. * @throws ArgumentNotValid If the replica is null. */ @Override public long getNumberOfFiles(Replica replica) throws ArgumentNotValid { ArgumentNotValid.checkNotNull(replica, "Replica replica"); Connection con = ArchiveDBConnection.get(); // The SQL statement to retrieve the amount of entries in the // replicafileinfo table for the replica which have the // filelist_status set to OK. String sql = "SELECT COUNT(*) FROM replicafileinfo WHERE replica_id = ? AND filelist_status = ?"; try { return DBUtils.selectLongValue(con, sql, replica.getId(), FileListStatus.OK.ordinal()); } finally { ArchiveDBConnection.release(con); } } /** * Method for finding a replica with a valid version of a file. This method is used in order to find a replica from * which a file should be retrieved, during the process of restoring a corrupt file on another replica. * <p> * This replica must of the type bitarchive, since a file cannot be retrieved from a checksum replica. * * @param filename The name of the file which needs to have a valid version in a bitarchive. * @return A bitarchive which contains a valid version of the file, or null if no such bitarchive exists. * @throws ArgumentNotValid If the filename is null or the empty string. */ @Override public Replica getBitarchiveWithGoodFile(String filename) throws ArgumentNotValid { ArgumentNotValid.checkNotNullOrEmpty(filename, "String filename"); Connection con = ArchiveDBConnection.get(); try { // Retrieve a list of replicas where the the checksum status is OK List<String> replicaIds = ReplicaCacheHelpers.retrieveReplicaIdsWithOKChecksumStatus(filename, con); // go through the list, and return the first valid bitarchive-replica. for (String repId : replicaIds) { // Retrieve the replica type. ReplicaType repType = ReplicaCacheHelpers.retrieveReplicaType(repId, con); // If the replica is of type BITARCHIVE then return it. if (repType.equals(ReplicaType.BITARCHIVE)) { log.trace( "The replica with id '{}' is the first bitarchive replica which contains the file '{}' " + "with a valid checksum.", repId, filename); return Replica.getReplicaFromId(repId); } } } finally { ArchiveDBConnection.release(con); } // Notify the administrator about that no proper bitarchive was found. NotificationsFactory.getInstance().notify( "No bitarchive replica " + "was found which contains the file '" + filename + "'.", NotificationType.WARNING); // If no bitarchive exists that contains the file with a OK checksum_status. // then return null. return null; } /** * Method for finding a replica with a valid version of a file. This method is used in order to find a replica from * which a file should be retrieved, during the process of restoring a corrupt file on another replica. * <p> * This replica must of the type bitarchive, since a file cannot be retrieved from a checksum replica. * * @param filename The name of the file which needs to have a valid version in a bitarchive. * @param badReplica The Replica which has a bad copy of the given file * @return A bitarchive which contains a valid version of the file, or null if no such bitarchive exists (in which * case, a notification is sent) * @throws ArgumentNotValid If the replica is null or the filename is either null or the empty string. */ @Override public Replica getBitarchiveWithGoodFile(String filename, Replica badReplica) throws ArgumentNotValid { ArgumentNotValid.checkNotNull(badReplica, "Replica badReplica"); ArgumentNotValid.checkNotNullOrEmpty(filename, "String filename"); Connection con = ArchiveDBConnection.get(); try { // Then retrieve a list of replicas where the the checksum status is // OK List<String> replicaIds = ReplicaCacheHelpers.retrieveReplicaIdsWithOKChecksumStatus(filename, con); // Make sure, that the bad replica is not returned. replicaIds.remove(badReplica.getId()); // go through the list, and return the first valid // bitarchive-replica. for (String repId : replicaIds) { // Retrieve the replica type. ReplicaType repType = ReplicaCacheHelpers.retrieveReplicaType(repId, con); // If the replica is of type BITARCHIVE then return it. if (repType.equals(ReplicaType.BITARCHIVE)) { log.trace( "The replica with id '{}' is the first bitarchive replica which contains the file '{}' with a valid checksum.", repId, filename); return Replica.getReplicaFromId(repId); } } } finally { ArchiveDBConnection.release(con); } // Notify the administrator about that no proper bitarchive was found, and log the incidence final String msg = "No bitarchive replica " + "was found which contains the file '" + filename + "'."; log.warn(msg); NotificationsFactory.getInstance().notify(msg, NotificationType.WARNING); return null; } /** * Method for updating a specific entry in the replicafileinfo table. Based on the filename, checksum and replica it * is verified whether a file is missing, corrupt or valid. * * @param filename Name of the file. * @param checksum The checksum of the file. Is allowed to be null, if no file is found. * @param replica The replica where the file exists. * @throws ArgumentNotValid If the filename is null or the empty string, or if the replica is null. */ @Override public void updateChecksumInformationForFileOnReplica(String filename, String checksum, Replica replica) throws ArgumentNotValid { ArgumentNotValid.checkNotNullOrEmpty(filename, "String filename"); ArgumentNotValid.checkNotNull(replica, "Replica replica"); PreparedStatement statement = null; Connection connection = null; try { connection = ArchiveDBConnection.get(); long guid = ReplicaCacheHelpers.retrieveGuidForFilenameOnReplica(filename, replica.getId(), connection); Date now = new Date(Calendar.getInstance().getTimeInMillis()); // handle differently whether a checksum was retrieved. if (checksum == null) { // Set to MISSING! and do not update the checksum // (cannot insert null). String sql = "UPDATE replicafileinfo " + "SET filelist_status = ?, checksum_status = ?, filelist_checkdatetime = ? " + "WHERE replicafileinfo_guid = ?"; statement = DBUtils.prepareStatement(connection, sql, FileListStatus.MISSING.ordinal(), ChecksumStatus.UNKNOWN.ordinal(), now, guid); } else { String sql = "UPDATE replicafileinfo " + "SET checksum = ?, filelist_status = ?, filelist_checkdatetime = ? " + "WHERE replicafileinfo_guid = ?"; statement = DBUtils.prepareStatement(connection, sql, checksum, FileListStatus.OK.ordinal(), now, guid); } statement.executeUpdate(); connection.commit(); } catch (Exception e) { throw new IOFailure("Could not update single checksum entry.", e); } finally { DBUtils.closeStatementIfOpen(statement); if (connection != null) { ArchiveDBConnection.release(connection); } } } /** * Method for inserting a line of Admin.Data into the database. It is assumed that it is a '0.4' admin.data line. * * @param line The line to insert into the database. * @return Whether the line was valid. * @throws ArgumentNotValid If the line is null. If it is empty, then it is logged. */ public boolean insertAdminEntry(String line) throws ArgumentNotValid { ArgumentNotValid.checkNotNull(line, "String line"); Connection con = ArchiveDBConnection.get(); log.trace("Insert admin entry begun"); final int lengthFirstPart = 4; final int lengthOtherParts = 3; try { // split into parts. First contains String[] split = line.split(" , "); // Retrieve the basic entry data. String[] entryData = split[0].split(" "); // Check if enough elements if (entryData.length < lengthFirstPart) { log.warn("Bad line in Admin.data: {}", line); return false; } String filename = entryData[0]; String checksum = entryData[1]; long fileId = ReplicaCacheHelpers.retrieveIdForFile(filename, con); // If the fileId is -1, then the file is not within the file table. // Thus insert it and retrieve the id. if (fileId == -1) { fileId = ReplicaCacheHelpers.insertFileIntoDB(filename, con); } log.trace("Step 1 completed (file created in database)."); // go through the replica specifics. for (int i = 1; i < split.length; i++) { String[] repInfo = split[i].split(" "); // check if correct size if (repInfo.length < lengthOtherParts) { log.warn("Bad replica information '{}' in line '{}'", split[i], line); continue; } // retrieve the data for this replica String replicaId = Channels.retrieveReplicaFromIdentifierChannel(repInfo[0]).getId(); ReplicaStoreState replicaUploadStatus = ReplicaStoreState.valueOf(repInfo[1]); Date replicaDate = new Date(Long.parseLong(repInfo[2])); // retrieve the guid of the replicafileinfo. long guid = ReplicaCacheHelpers.retrieveReplicaFileInfoGuid(fileId, replicaId, con); // Update the replicaFileInfo with the information. ReplicaCacheHelpers.updateReplicaFileInfo(guid, checksum, replicaDate, replicaUploadStatus, con); } } catch (IllegalState e) { log.warn("Received IllegalState exception while parsing.", e); return false; } finally { ArchiveDBConnection.release(con); } log.trace("Insert admin entry finished"); return true; } /** * Method for setting a specific value for the filelistdate and the checksumlistdate for all the replicas. * * @param date The new date for the checksumlist and filelist for all the replicas. * @throws ArgumentNotValid If the date is null. */ public void setAdminDate(Date date) throws ArgumentNotValid { ArgumentNotValid.checkNotNull(date, "Date date"); Connection con = ArchiveDBConnection.get(); try { // set the date for the replicas. for (Replica rep : Replica.getKnown()) { ReplicaCacheHelpers.setFilelistDateForReplica(rep, date, con); ReplicaCacheHelpers.setChecksumlistDateForReplica(rep, date, con); } } finally { ArchiveDBConnection.release(con); } } /** * Method for telling whether the database is empty. The database is empty if it does not contain any files. * <p> * The database will not be entirely empty, since the replicas are put into the replica table during the * instantiation of this class, but if the file table is empty, then the replicafileinfo table is also empty, and * the database will be considered empty. * * @return Whether the file list is empty. */ public boolean isEmpty() { // The SQL statement to retrieve the amount of entries in the // file table. No arguments (represented by empty Object array). final String sql = "SELECT COUNT(*) FROM file"; Connection con = ArchiveDBConnection.get(); try { return DBUtils.selectLongValue(con, sql, new Object[0]) == 0L; } finally { ArchiveDBConnection.release(con); } } /** * Method to print all the tables in the database. * * @return all the tables as a text string */ public String retrieveAsText() { StringBuilder res = new StringBuilder(); String sql = ""; Connection connection = ArchiveDBConnection.get(); // Go through the replica table List<String> reps = ReplicaCacheHelpers.retrieveIdsFromReplicaTable(connection); res.append("Replica table: " + reps.size() + "\n"); res.append("GUID \trepId \trepName \trepType \tfileupdate \tchecksumupdated" + "\n"); res.append("------------------------------------------------------------\n"); for (String repId : reps) { // retrieve the replica_name sql = "SELECT replica_guid FROM replica WHERE replica_id = ?"; String repGUID = DBUtils.selectStringValue(connection, sql, repId); // retrieve the replica_name sql = "SELECT replica_name FROM replica WHERE replica_id = ?"; String repName = DBUtils.selectStringValue(connection, sql, repId); // retrieve the replica_type sql = "SELECT replica_type FROM replica WHERE replica_id = ?"; int repType = DBUtils.selectIntValue(connection, sql, repId); // retrieve the date for last updated sql = "SELECT filelist_updated FROM replica WHERE replica_id = ?"; String filelistUpdated = DBUtils.selectStringValue(connection, sql, repId); // retrieve the date for last updated sql = "SELECT checksum_updated FROM replica WHERE replica_id = ?"; String checksumUpdated = DBUtils.selectStringValue(connection, sql, repId); // Print res.append(repGUID + "\t" + repId + "\t" + repName + "\t" + ReplicaType.fromOrdinal(repType).name() + "\t" + filelistUpdated + "\t" + checksumUpdated + "\n"); } res.append("\n"); // Go through the file table List<String> fileIds = ReplicaCacheHelpers.retrieveIdsFromFileTable(connection); res.append("File table : " + fileIds.size() + "\n"); res.append("fileId \tfilename" + "\n"); res.append("--------------------" + "\n"); for (String fileId : fileIds) { // retrieve the file_name sql = "SELECT filename FROM file WHERE file_id = ?"; String fileName = DBUtils.selectStringValue(connection, sql, fileId); // Print res.append(fileId + " \t " + fileName + "\n"); } res.append("\n"); // Go through the replicafileinfo table List<String> rfiIds = ReplicaCacheHelpers.retrieveIdsFromReplicaFileInfoTable(connection); res.append("ReplicaFileInfo table : " + rfiIds.size() + "\n"); res.append("GUID \trepId \tfileId \tchecksum \tus \t\tfls \tcss \tfilelistCheckdate \tchecksumCheckdate\n"); res.append( "---------------------------------------------------------------------------------------------------------\n"); for (String rfiGUID : rfiIds) { // FIXME Replace with one SELECT instead of one SELECT for each row! DOH! // retrieve the replica_id sql = "SELECT replica_id FROM replicafileinfo WHERE replicafileinfo_guid = ?"; String replicaId = DBUtils.selectStringValue(connection, sql, rfiGUID); // retrieve the file_id sql = "SELECT file_id FROM replicafileinfo WHERE replicafileinfo_guid = ?"; String fileId = DBUtils.selectStringValue(connection, sql, rfiGUID); // retrieve the checksum sql = "SELECT checksum FROM replicafileinfo WHERE replicafileinfo_guid = ?"; String checksum = DBUtils.selectStringValue(connection, sql, rfiGUID); // retrieve the upload_status sql = "SELECT upload_status FROM replicafileinfo WHERE replicafileinfo_guid = ?"; int uploadStatus = DBUtils.selectIntValue(connection, sql, rfiGUID); // retrieve the filelist_status sql = "SELECT filelist_status FROM replicafileinfo WHERE replicafileinfo_guid = ?"; int filelistStatus = DBUtils.selectIntValue(connection, sql, rfiGUID); // retrieve the checksum_status sql = "SELECT checksum_status FROM replicafileinfo WHERE replicafileinfo_guid = ?"; int checksumStatus = DBUtils.selectIntValue(connection, sql, rfiGUID); // retrieve the filelist_checkdatetime sql = "SELECT filelist_checkdatetime FROM replicafileinfo WHERE replicafileinfo_guid = ?"; String filelistCheckdatetime = DBUtils.selectStringValue(connection, sql, rfiGUID); // retrieve the checksum_checkdatetime sql = "SELECT checksum_checkdatetime FROM replicafileinfo WHERE replicafileinfo_guid = ?"; String checksumCheckdatetime = DBUtils.selectStringValue(connection, sql, rfiGUID); // Print res.append(rfiGUID + " \t" + replicaId + "\t" + fileId + "\t" + checksum + "\t" + ReplicaStoreState.fromOrdinal(uploadStatus).name() + " \t" + FileListStatus.fromOrdinal(filelistStatus).name() + "\t" + ChecksumStatus.fromOrdinal(checksumStatus).name() + "\t" + filelistCheckdatetime + "\t" + checksumCheckdatetime + "\n"); } res.append("\n"); return res.toString(); } /** * Method for cleaning up. */ @Override public synchronized void cleanup() { instance = null; } }