com.archivas.clienttools.arcutils.utils.database.PopulateDmDb.java Source code

Java tutorial

Introduction

Here is the source code for com.archivas.clienttools.arcutils.utils.database.PopulateDmDb.java

Source

// Copyright 2007 Hitachi Data Systems
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package com.archivas.clienttools.arcutils.utils.database;

import com.archivas.clienttools.arcutils.api.ArcMoverEngine;
import com.archivas.clienttools.arcutils.api.JobException;
import com.archivas.clienttools.arcutils.api.JobId;
import com.archivas.clienttools.arcutils.api.jobs.ManagedJob;
import com.archivas.clienttools.arcutils.api.jobs.ManagedJobSummary;
import com.archivas.clienttools.arcutils.impl.LocalJvmArcMoverEngine;
import com.archivas.clienttools.arcutils.impl.jobs.FileStatus;
import com.archivas.clienttools.arcutils.impl.jobs.ManagedJobImpl;
import com.archivas.clienttools.arcutils.profile.AbstractProfileBase;
import com.archivas.clienttools.arcutils.profile.ProfileManager;
import com.archivas.clienttools.arcutils.model.ArcCopyFile;
import com.archivas.clienttools.arcutils.model.ArcMoverFile;
import com.archivas.clienttools.arcutils.model.FileMetadata;
import com.archivas.clienttools.arcutils.model.FileType;
import com.archivas.clienttools.arcutils.utils.FileUtil;
import com.archivas.clienttools.arcutils.utils.net.SSLCertChain;
import com.archivas.clienttools.arcutils.utils.net.SSLCertificateCallback;
import org.apache.commons.lang.time.DurationFormatUtils;

import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.List;

public class PopulateDmDb implements SSLCertificateCallback {

    private String jobName;
    private ManagedJob.Type jobType;
    private long rowCnt;
    private long startTime = 0;
    private int percentReadyToFind;
    private int percentReadyToProcess;
    private int percentReadyToPostProcess;
    private long readyToFindCnt;
    private long readyToProcessCnt;
    private long readyToPostProcessCnt;
    private FileLifeCycle jobLifeCycle;
    private FileLifeCycle fileLifeCycle;
    private FileStatus fileStatus;

    private static final String COPY_INSERT_COLS_SQL = "(INITIAL_LIST, " + // 1
            "RECORD_ID, " + // 2
            "RECORD_TYPE, " + // 3
            "SOURCE_PATH, " + // 4
            "SOURCE_VERSION, " + // 5
            "SOURCE_PROFILE_NAME, " + // 6
            "LIFE_CYCLE, " + // 7
            "STATUS, " + // 8
            "TARGET_PATH, " + // 9
            "TARGET_PROFILE_NAME, " + // 10
            "SIZE)" + // 11
            " VALUES " + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";

    private static final String DELETE_INSERT_COLS_SQL = "(INITIAL_LIST, " + // 1
            "RECORD_ID, " + // 2
            "RECORD_TYPE, " + // 3
            "SOURCE_PATH, " + // 4
            "SOURCE_VERSION, " + // 5
            "SOURCE_PROFILE_NAME, " + // 6
            "LIFE_CYCLE, " + // 7
            "STATUS) " + // 8
            " VALUES " + "(?, ?, ?, ?, ?, ?, ?, ?)";

    private PopulateDmDb(String jobName, ManagedJob.Type jobType, long rowCnt, int percentReadyToFind,
            int percentReadyToProcess, int percentReadyToPostProcess, FileLifeCycle jobLifeCycle) {
        this.jobName = jobName;
        this.jobType = jobType;
        this.rowCnt = rowCnt;
        this.percentReadyToFind = percentReadyToFind;
        this.percentReadyToProcess = percentReadyToProcess;
        this.percentReadyToPostProcess = percentReadyToPostProcess;
        this.jobLifeCycle = jobLifeCycle;
    }

    private void insertRows() throws SQLException, DatabaseException, JobException {
        ArcMoverEngine engine = LocalJvmArcMoverEngine.getInstance();

        List<ManagedJobSummary> allJobs = engine.getAllManagedJobs();
        JobId jobId = null;
        for (ManagedJobSummary summary : allJobs) {
            if (summary.getJobName().equals(jobName)) {
                jobId = summary.getJobId();
                break;
            }
        }
        if (jobId == null) {
            throw new IllegalArgumentException("Job \"" + jobName + "\" not found");
        }

        System.out.print("Loading managed job ....");
        System.out.flush();
        ManagedJobImpl jobImpl = engine.loadManagedJob(jobId, jobType);
        System.out.println(" done");

        ManagedJob job = jobImpl.getJob();
        AbstractProfileBase sourceProfile = job.getSourceProfile();
        AbstractProfileBase targetProfile = job.getTargetProfile();
        String sourcePath = job.getSourcePath();
        String targetPath = job.getTargetPath();

        ArcCopyFile file = generateFile(sourceProfile, targetProfile, sourcePath, targetPath);
        String fileSourcePath = file.getSourcePath();
        String fileSourceProfile = file.getSourceProfile().getName();
        String fileTargetPath = file.getTargetPath();
        String fileTargetProfile = (targetProfile == null ? null : file.getTargetProfile().getName());
        long fileSize = file.getSize();
        int fileObjectType = FileType.UNKNOWN.ordinal();
        if (file.isDirectory()) {
            fileObjectType = FileType.DIRECTORY.ordinal();
        } else if (file.isFile()) {
            fileObjectType = FileType.FILE.ordinal();
        }
        Long version = file.getSourceVersion();
        if (version == null) {
            version = 0L;
        }

        long recordId = jobImpl.getManagedJobSchema().getLastDbRecordId();
        long discoveredObjectCnt = jobImpl.getDiscoveredObjectCount();
        long totalObjCnt = jobImpl.getTotalObjectCount();

        System.out.println(
                "*** max RECORD_ID = " + recordId + ", initialDiscoveredObjectCnt = " + discoveredObjectCnt);

        boolean isDelete = (jobType == ManagedJob.Type.DELETE);
        String insertSql = "INSERT INTO " + ManagedJobSchema.getJobSchemaName(jobId.getId()) + "."
                + ManagedJobSchema.JOB_FILES_TABLE_NAME
                + (isDelete ? DELETE_INSERT_COLS_SQL : COPY_INSERT_COLS_SQL);
        String updateSql = "UPDATE " + ManagedJobsSchema.JOBS_TABLE_NAME + " set "
                + ManagedJobTableColumn.DISCOVERED_OBJ_CNT + " = ?, " + // 1
                ManagedJobTableColumn.MAX_RECORD_ID + " = ?, " + // 2
                ManagedJobTableColumn.TOTAL_OBJ_CNT + " = ? " + // 3
                " WHERE " + ManagedJobTableColumn.JOB_ID + " = ?"; // 4

        Connection conn = DatabaseResourceManager.createConnection();
        conn.setAutoCommit(false);
        PreparedStatement insertStatement = conn.prepareStatement(insertSql);
        PreparedStatement updateStatement = conn.prepareStatement(updateSql);

        startTime = System.currentTimeMillis();
        long l = 0;
        for (; l < rowCnt; l++) {
            recordId++;
            getLifeCycle(l);
            if (fileLifeCycle.ordinal() > FileLifeCycle.FINDING.ordinal()) {
                totalObjCnt++;
            }
            discoveredObjectCnt++;

            insertStatement.clearParameters();
            insertStatement.setInt(1, DatabaseResourceManager.boolToDbValue(false)); // initial list
            insertStatement.setLong(2, recordId); // record id

            insertStatement.setInt(3, fileObjectType); // record type
            insertStatement.setString(4, fileSourcePath); // source path
            insertStatement.setLong(5, version); // source version
            insertStatement.setString(6, fileSourceProfile); // source profile name
            insertStatement.setInt(7, fileLifeCycle.ordinal()); // life cycle
            if (fileStatus == null) {
                insertStatement.setNull(8, java.sql.Types.SMALLINT); // status
            } else {
                insertStatement.setInt(8, fileStatus.ordinal());
            }

            if (!isDelete) {
                insertStatement.setString(9, fileTargetPath); // target path
                insertStatement.setString(10, fileTargetProfile); // target profile name
                insertStatement.setLong(11, fileSize); // size
            }

            insertStatement.execute();
            if (l % 5000 == 0) {
                // update counts in jobs table
                updateStatement.clearParameters();
                updateStatement.setLong(1, discoveredObjectCnt);
                updateStatement.setLong(2, recordId);
                updateStatement.setLong(3, totalObjCnt);
                updateStatement.setLong(4, jobId.getId());
                updateStatement.execute();

                conn.commit();
                displayStats(l);
            }
        }

        updateStatement.clearParameters();
        updateStatement.setLong(1, discoveredObjectCnt);
        updateStatement.setLong(2, recordId);
        updateStatement.setLong(3, totalObjCnt);
        updateStatement.setLong(4, jobId.getId());
        updateStatement.execute();

        conn.commit();
        displayStats(l);
    }

    private void getLifeCycle(long rowNum) {

        int val = (int) rowNum % 100;
        if (percentReadyToFind > 0 && val < percentReadyToFind) {
            // A val from 0 to percentReadyToFind will be marked ReadyToFind
            readyToFindCnt++;
            if (jobLifeCycle.ordinal() > FileLifeCycle.FINDING.ordinal()) {
                fileLifeCycle = FileLifeCycle.FINDING;
                fileStatus = FileStatus.FAILED;
            } else {
                fileLifeCycle = FileLifeCycle.READY_TO_FIND;
                fileStatus = FileStatus.NONE;
            }
        } else if (percentReadyToProcess > 0 && val >= percentReadyToFind
                && val < (percentReadyToFind + percentReadyToProcess)) {
            // A val from percentReadyToFind to (percentReadyToFind + percentReadyToProcess) will be
            // marked ReadyToProcess
            readyToProcessCnt++;
            if (jobLifeCycle.ordinal() > FileLifeCycle.PROCESSING.ordinal()) {
                fileLifeCycle = FileLifeCycle.PROCESSING;
                fileStatus = FileStatus.FAILED;
            } else {
                fileLifeCycle = FileLifeCycle.READY_TO_PROCESS;
                fileStatus = FileStatus.NONE;
            }
        } else if (percentReadyToPostProcess > 0 && val >= (percentReadyToFind + percentReadyToProcess)) {
            readyToPostProcessCnt++;
            if (jobLifeCycle.ordinal() > FileLifeCycle.POSTPROCESSING.ordinal()) {
                fileLifeCycle = FileLifeCycle.POSTPROCESSING;
                fileStatus = FileStatus.FAILED;
            } else {
                fileLifeCycle = FileLifeCycle.READY_TO_POSTPROCESS;
                fileStatus = FileStatus.NONE;
            }
        } else {
            throw new RuntimeException("Could not determine the life cycle to get");
        }
    }

    private void displayStats(long insertedCnt) {
        float percentComplete = (float) insertedCnt / (float) rowCnt;
        long now = System.currentTimeMillis();
        long elapsedTime = now - startTime;
        long timeRemaining = ((long) (((float) (now - startTime)) / percentComplete)) - elapsedTime;
        String elapsedTimeStr = DurationFormatUtils.formatDurationWords(elapsedTime, true, true);
        String timeRemainingStr = DurationFormatUtils.formatDurationWords(timeRemaining, true, true);

        System.out.println("Inserted " + insertedCnt + " rows (" + readyToFindCnt + " readyToFind, "
                + readyToProcessCnt + " readyToProcess, " + readyToPostProcessCnt + " readyToPostProcess) in "
                + elapsedTimeStr + ".  " + ((int) (percentComplete * 100)) + " % complete, " + timeRemainingStr
                + " remaining.");
    }

    private ArcCopyFile generateFile(AbstractProfileBase sourceProfile, AbstractProfileBase targetProfile,
            String rootSourcePath, String rootTargetPath) {
        String fileName = generateFilePath();
        fileName = FileUtil.converWindowsPathToUnix(fileName, sourceProfile.getPathSeparator());

        // It is possible on windows that both the base path and the file path have C: in it.
        // Remove those before proceeding
        String sourceFileName = fileName;
        if (sourceFileName.indexOf(":") == 1 && rootSourcePath.indexOf(":") == 1) {
            sourceFileName = sourceFileName.substring(2);
        }
        String targetFileName = fileName;
        if (fileName.indexOf(":") == 1) {
            targetFileName = targetFileName.substring(2);
        }

        String completeSourcePath = FileUtil.resolvePath(rootSourcePath, sourceFileName,
                sourceProfile.getPathSeparator());

        // Not here for delete jobs and we are reusing that code here
        String completeTargetPath = null;
        if (targetProfile != null) {
            completeTargetPath = FileUtil.resolvePath(rootTargetPath, targetFileName,
                    targetProfile.getPathSeparator());
        }

        ArcMoverFile source = ArcMoverFile.getFileInstance(sourceProfile, completeSourcePath,
                new FileMetadata(FileType.FILE));
        return new ArcCopyFile(source, targetProfile, completeTargetPath, null);
    }

    private String generateFilePath() {
        return "abcd/efg/hijkl/mnop/qrstuv/wxyzabcd/efg/hijk/lmnop";
    }

    public void validCertCallback(AbstractProfileBase profile, SSLCertChain certChain) {
        profile.setSSLCertChain(certChain);
    }

    public SSLCertificateCallback.AllowSSLCert exceptionCallback(AbstractProfileBase profile,
            SSLCertChain certChain, String warningString) {
        return SSLCertificateCallback.AllowSSLCert.THIS_SESSION_ONLY;
    }

    public static void main(String[] args) {
        String usageString = "Usage: hcpdm populate <job_name> <COPY|DELETE> <rowCnt> <percentReadyToFind> <percentReadyToProcess> <percentReadyToPostProcess> <life-cycle>";

        String jobName = null;
        long rowCnt = 0L;
        ManagedJob.Type jobType = null;
        int percentReadyToFind = 0;
        int percentReadyToProcess = 0;
        int percentReadyToPostProcess = 0;
        FileLifeCycle lifeCycle = null;

        try {
            jobName = args[1];
            jobType = ManagedJob.Type.valueOf(args[2]);
            if (jobType == null) {
                throw new IllegalArgumentException("invalid job type <" + args[2] + ">.  Must be COPY or DELETE");
            }
            rowCnt = Long.valueOf(args[3]);
            percentReadyToFind = Integer.valueOf(args[4]);
            percentReadyToProcess = Integer.valueOf(args[5]);
            percentReadyToPostProcess = Integer.valueOf(args[6]);
            if (percentReadyToFind + percentReadyToProcess + percentReadyToPostProcess != 100) {
                throw new IllegalArgumentException(
                        "percentReadyToFind + percentReadyToProcess + percentReadyToPostProcess must equal 100");
            }
            if (percentReadyToFind < 0 || percentReadyToProcess < 0 || percentReadyToPostProcess < 0) {
                throw new IllegalArgumentException(
                        "percentReadyToFind, percentReadyToProcess, and percentReadyToPostProcess must all be between 0 and 100, inclusive");
            }
            lifeCycle = FileLifeCycle.valueOf(args[7]);
        } catch (Exception e) {
            System.out.println(usageString);
            e.printStackTrace();
            System.exit(-1);
        }

        try {
            PopulateDmDb pop = new PopulateDmDb(jobName, jobType, rowCnt, percentReadyToFind, percentReadyToProcess,
                    percentReadyToPostProcess, lifeCycle);
            ProfileManager.initialize(pop);
            // pop.populateDb();
            pop.insertRows();

        } catch (Exception e) {
            e.printStackTrace();
        }
    }

}