Java tutorial
// Copyright 2007 Hitachi Data Systems // All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); you may // not use this file except in compliance with the License. You may obtain // a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the // License for the specific language governing permissions and limitations // under the License. package com.archivas.clienttools.arcutils.utils.database; import com.archivas.clienttools.arcutils.api.ArcMoverEngine; import com.archivas.clienttools.arcutils.api.JobException; import com.archivas.clienttools.arcutils.api.JobId; import com.archivas.clienttools.arcutils.api.jobs.ManagedJob; import com.archivas.clienttools.arcutils.api.jobs.ManagedJobSummary; import com.archivas.clienttools.arcutils.impl.LocalJvmArcMoverEngine; import com.archivas.clienttools.arcutils.impl.jobs.FileStatus; import com.archivas.clienttools.arcutils.impl.jobs.ManagedJobImpl; import com.archivas.clienttools.arcutils.profile.AbstractProfileBase; import com.archivas.clienttools.arcutils.profile.ProfileManager; import com.archivas.clienttools.arcutils.model.ArcCopyFile; import com.archivas.clienttools.arcutils.model.ArcMoverFile; import com.archivas.clienttools.arcutils.model.FileMetadata; import com.archivas.clienttools.arcutils.model.FileType; import com.archivas.clienttools.arcutils.utils.FileUtil; import com.archivas.clienttools.arcutils.utils.net.SSLCertChain; import com.archivas.clienttools.arcutils.utils.net.SSLCertificateCallback; import org.apache.commons.lang.time.DurationFormatUtils; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.List; public class PopulateDmDb implements SSLCertificateCallback { private String jobName; private ManagedJob.Type jobType; private long rowCnt; private long startTime = 0; private int percentReadyToFind; private int percentReadyToProcess; private int percentReadyToPostProcess; private long readyToFindCnt; private long readyToProcessCnt; private long readyToPostProcessCnt; private FileLifeCycle jobLifeCycle; private FileLifeCycle fileLifeCycle; private FileStatus fileStatus; private static final String COPY_INSERT_COLS_SQL = "(INITIAL_LIST, " + // 1 "RECORD_ID, " + // 2 "RECORD_TYPE, " + // 3 "SOURCE_PATH, " + // 4 "SOURCE_VERSION, " + // 5 "SOURCE_PROFILE_NAME, " + // 6 "LIFE_CYCLE, " + // 7 "STATUS, " + // 8 "TARGET_PATH, " + // 9 "TARGET_PROFILE_NAME, " + // 10 "SIZE)" + // 11 " VALUES " + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; private static final String DELETE_INSERT_COLS_SQL = "(INITIAL_LIST, " + // 1 "RECORD_ID, " + // 2 "RECORD_TYPE, " + // 3 "SOURCE_PATH, " + // 4 "SOURCE_VERSION, " + // 5 "SOURCE_PROFILE_NAME, " + // 6 "LIFE_CYCLE, " + // 7 "STATUS) " + // 8 " VALUES " + "(?, ?, ?, ?, ?, ?, ?, ?)"; private PopulateDmDb(String jobName, ManagedJob.Type jobType, long rowCnt, int percentReadyToFind, int percentReadyToProcess, int percentReadyToPostProcess, FileLifeCycle jobLifeCycle) { this.jobName = jobName; this.jobType = jobType; this.rowCnt = rowCnt; this.percentReadyToFind = percentReadyToFind; this.percentReadyToProcess = percentReadyToProcess; this.percentReadyToPostProcess = percentReadyToPostProcess; this.jobLifeCycle = jobLifeCycle; } private void insertRows() throws SQLException, DatabaseException, JobException { ArcMoverEngine engine = LocalJvmArcMoverEngine.getInstance(); List<ManagedJobSummary> allJobs = engine.getAllManagedJobs(); JobId jobId = null; for (ManagedJobSummary summary : allJobs) { if (summary.getJobName().equals(jobName)) { jobId = summary.getJobId(); break; } } if (jobId == null) { throw new IllegalArgumentException("Job \"" + jobName + "\" not found"); } System.out.print("Loading managed job ...."); System.out.flush(); ManagedJobImpl jobImpl = engine.loadManagedJob(jobId, jobType); System.out.println(" done"); ManagedJob job = jobImpl.getJob(); AbstractProfileBase sourceProfile = job.getSourceProfile(); AbstractProfileBase targetProfile = job.getTargetProfile(); String sourcePath = job.getSourcePath(); String targetPath = job.getTargetPath(); ArcCopyFile file = generateFile(sourceProfile, targetProfile, sourcePath, targetPath); String fileSourcePath = file.getSourcePath(); String fileSourceProfile = file.getSourceProfile().getName(); String fileTargetPath = file.getTargetPath(); String fileTargetProfile = (targetProfile == null ? null : file.getTargetProfile().getName()); long fileSize = file.getSize(); int fileObjectType = FileType.UNKNOWN.ordinal(); if (file.isDirectory()) { fileObjectType = FileType.DIRECTORY.ordinal(); } else if (file.isFile()) { fileObjectType = FileType.FILE.ordinal(); } Long version = file.getSourceVersion(); if (version == null) { version = 0L; } long recordId = jobImpl.getManagedJobSchema().getLastDbRecordId(); long discoveredObjectCnt = jobImpl.getDiscoveredObjectCount(); long totalObjCnt = jobImpl.getTotalObjectCount(); System.out.println( "*** max RECORD_ID = " + recordId + ", initialDiscoveredObjectCnt = " + discoveredObjectCnt); boolean isDelete = (jobType == ManagedJob.Type.DELETE); String insertSql = "INSERT INTO " + ManagedJobSchema.getJobSchemaName(jobId.getId()) + "." + ManagedJobSchema.JOB_FILES_TABLE_NAME + (isDelete ? DELETE_INSERT_COLS_SQL : COPY_INSERT_COLS_SQL); String updateSql = "UPDATE " + ManagedJobsSchema.JOBS_TABLE_NAME + " set " + ManagedJobTableColumn.DISCOVERED_OBJ_CNT + " = ?, " + // 1 ManagedJobTableColumn.MAX_RECORD_ID + " = ?, " + // 2 ManagedJobTableColumn.TOTAL_OBJ_CNT + " = ? " + // 3 " WHERE " + ManagedJobTableColumn.JOB_ID + " = ?"; // 4 Connection conn = DatabaseResourceManager.createConnection(); conn.setAutoCommit(false); PreparedStatement insertStatement = conn.prepareStatement(insertSql); PreparedStatement updateStatement = conn.prepareStatement(updateSql); startTime = System.currentTimeMillis(); long l = 0; for (; l < rowCnt; l++) { recordId++; getLifeCycle(l); if (fileLifeCycle.ordinal() > FileLifeCycle.FINDING.ordinal()) { totalObjCnt++; } discoveredObjectCnt++; insertStatement.clearParameters(); insertStatement.setInt(1, DatabaseResourceManager.boolToDbValue(false)); // initial list insertStatement.setLong(2, recordId); // record id insertStatement.setInt(3, fileObjectType); // record type insertStatement.setString(4, fileSourcePath); // source path insertStatement.setLong(5, version); // source version insertStatement.setString(6, fileSourceProfile); // source profile name insertStatement.setInt(7, fileLifeCycle.ordinal()); // life cycle if (fileStatus == null) { insertStatement.setNull(8, java.sql.Types.SMALLINT); // status } else { insertStatement.setInt(8, fileStatus.ordinal()); } if (!isDelete) { insertStatement.setString(9, fileTargetPath); // target path insertStatement.setString(10, fileTargetProfile); // target profile name insertStatement.setLong(11, fileSize); // size } insertStatement.execute(); if (l % 5000 == 0) { // update counts in jobs table updateStatement.clearParameters(); updateStatement.setLong(1, discoveredObjectCnt); updateStatement.setLong(2, recordId); updateStatement.setLong(3, totalObjCnt); updateStatement.setLong(4, jobId.getId()); updateStatement.execute(); conn.commit(); displayStats(l); } } updateStatement.clearParameters(); updateStatement.setLong(1, discoveredObjectCnt); updateStatement.setLong(2, recordId); updateStatement.setLong(3, totalObjCnt); updateStatement.setLong(4, jobId.getId()); updateStatement.execute(); conn.commit(); displayStats(l); } private void getLifeCycle(long rowNum) { int val = (int) rowNum % 100; if (percentReadyToFind > 0 && val < percentReadyToFind) { // A val from 0 to percentReadyToFind will be marked ReadyToFind readyToFindCnt++; if (jobLifeCycle.ordinal() > FileLifeCycle.FINDING.ordinal()) { fileLifeCycle = FileLifeCycle.FINDING; fileStatus = FileStatus.FAILED; } else { fileLifeCycle = FileLifeCycle.READY_TO_FIND; fileStatus = FileStatus.NONE; } } else if (percentReadyToProcess > 0 && val >= percentReadyToFind && val < (percentReadyToFind + percentReadyToProcess)) { // A val from percentReadyToFind to (percentReadyToFind + percentReadyToProcess) will be // marked ReadyToProcess readyToProcessCnt++; if (jobLifeCycle.ordinal() > FileLifeCycle.PROCESSING.ordinal()) { fileLifeCycle = FileLifeCycle.PROCESSING; fileStatus = FileStatus.FAILED; } else { fileLifeCycle = FileLifeCycle.READY_TO_PROCESS; fileStatus = FileStatus.NONE; } } else if (percentReadyToPostProcess > 0 && val >= (percentReadyToFind + percentReadyToProcess)) { readyToPostProcessCnt++; if (jobLifeCycle.ordinal() > FileLifeCycle.POSTPROCESSING.ordinal()) { fileLifeCycle = FileLifeCycle.POSTPROCESSING; fileStatus = FileStatus.FAILED; } else { fileLifeCycle = FileLifeCycle.READY_TO_POSTPROCESS; fileStatus = FileStatus.NONE; } } else { throw new RuntimeException("Could not determine the life cycle to get"); } } private void displayStats(long insertedCnt) { float percentComplete = (float) insertedCnt / (float) rowCnt; long now = System.currentTimeMillis(); long elapsedTime = now - startTime; long timeRemaining = ((long) (((float) (now - startTime)) / percentComplete)) - elapsedTime; String elapsedTimeStr = DurationFormatUtils.formatDurationWords(elapsedTime, true, true); String timeRemainingStr = DurationFormatUtils.formatDurationWords(timeRemaining, true, true); System.out.println("Inserted " + insertedCnt + " rows (" + readyToFindCnt + " readyToFind, " + readyToProcessCnt + " readyToProcess, " + readyToPostProcessCnt + " readyToPostProcess) in " + elapsedTimeStr + ". " + ((int) (percentComplete * 100)) + " % complete, " + timeRemainingStr + " remaining."); } private ArcCopyFile generateFile(AbstractProfileBase sourceProfile, AbstractProfileBase targetProfile, String rootSourcePath, String rootTargetPath) { String fileName = generateFilePath(); fileName = FileUtil.converWindowsPathToUnix(fileName, sourceProfile.getPathSeparator()); // It is possible on windows that both the base path and the file path have C: in it. // Remove those before proceeding String sourceFileName = fileName; if (sourceFileName.indexOf(":") == 1 && rootSourcePath.indexOf(":") == 1) { sourceFileName = sourceFileName.substring(2); } String targetFileName = fileName; if (fileName.indexOf(":") == 1) { targetFileName = targetFileName.substring(2); } String completeSourcePath = FileUtil.resolvePath(rootSourcePath, sourceFileName, sourceProfile.getPathSeparator()); // Not here for delete jobs and we are reusing that code here String completeTargetPath = null; if (targetProfile != null) { completeTargetPath = FileUtil.resolvePath(rootTargetPath, targetFileName, targetProfile.getPathSeparator()); } ArcMoverFile source = ArcMoverFile.getFileInstance(sourceProfile, completeSourcePath, new FileMetadata(FileType.FILE)); return new ArcCopyFile(source, targetProfile, completeTargetPath, null); } private String generateFilePath() { return "abcd/efg/hijkl/mnop/qrstuv/wxyzabcd/efg/hijk/lmnop"; } public void validCertCallback(AbstractProfileBase profile, SSLCertChain certChain) { profile.setSSLCertChain(certChain); } public SSLCertificateCallback.AllowSSLCert exceptionCallback(AbstractProfileBase profile, SSLCertChain certChain, String warningString) { return SSLCertificateCallback.AllowSSLCert.THIS_SESSION_ONLY; } public static void main(String[] args) { String usageString = "Usage: hcpdm populate <job_name> <COPY|DELETE> <rowCnt> <percentReadyToFind> <percentReadyToProcess> <percentReadyToPostProcess> <life-cycle>"; String jobName = null; long rowCnt = 0L; ManagedJob.Type jobType = null; int percentReadyToFind = 0; int percentReadyToProcess = 0; int percentReadyToPostProcess = 0; FileLifeCycle lifeCycle = null; try { jobName = args[1]; jobType = ManagedJob.Type.valueOf(args[2]); if (jobType == null) { throw new IllegalArgumentException("invalid job type <" + args[2] + ">. Must be COPY or DELETE"); } rowCnt = Long.valueOf(args[3]); percentReadyToFind = Integer.valueOf(args[4]); percentReadyToProcess = Integer.valueOf(args[5]); percentReadyToPostProcess = Integer.valueOf(args[6]); if (percentReadyToFind + percentReadyToProcess + percentReadyToPostProcess != 100) { throw new IllegalArgumentException( "percentReadyToFind + percentReadyToProcess + percentReadyToPostProcess must equal 100"); } if (percentReadyToFind < 0 || percentReadyToProcess < 0 || percentReadyToPostProcess < 0) { throw new IllegalArgumentException( "percentReadyToFind, percentReadyToProcess, and percentReadyToPostProcess must all be between 0 and 100, inclusive"); } lifeCycle = FileLifeCycle.valueOf(args[7]); } catch (Exception e) { System.out.println(usageString); e.printStackTrace(); System.exit(-1); } try { PopulateDmDb pop = new PopulateDmDb(jobName, jobType, rowCnt, percentReadyToFind, percentReadyToProcess, percentReadyToPostProcess, lifeCycle); ProfileManager.initialize(pop); // pop.populateDb(); pop.insertRows(); } catch (Exception e) { e.printStackTrace(); } } }