asl.seedscan.SeedScan.java Source code

Java tutorial

Introduction

Here is the source code for asl.seedscan.SeedScan.java

Source

/*
 * Copyright 2011, United States Geological Survey or
 * third-party contributors as indicated by the @author tags.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/  >.
 *
 */
package asl.seedscan;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Set;

import java.io.*;

import java.io.BufferedInputStream;
import java.io.Console;
import java.io.DataInputStream;
import java.io.File;
import java.io.FilenameFilter;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Enumeration;
import java.util.Formatter;
import java.util.GregorianCalendar;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionGroup;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;

import asl.metadata.*;
import asl.metadata.meta_new.*;
import asl.security.*;
import asl.seedscan.config.*;
import asl.seedscan.database.*;
import asl.seedscan.metrics.*;
import asl.util.*;

/**
 * 
 */
public class SeedScan {
    private static Logger logger = LoggerFactory.getLogger(asl.seedscan.SeedScan.class);

    public static void main(String args[]) {
        // Default locations of config and schema files
        File configFile = new File("config.xml");
        File schemaFile = new File("schemas/SeedScanConfig.xsd");
        boolean parseConfig = true;

        ArrayList<File> schemaFiles = new ArrayList<File>();
        schemaFiles.add(schemaFile);

        // ==== Command Line Parsing ====
        Options options = new Options();
        Option opConfigFile = new Option("c", "config-file", true,
                "The config file to use for seedscan. XML format according to SeedScanConfig.xsd.");
        Option opSchemaFile = new Option("s", "schema-file", true,
                "The xsd schema file which should be used to verify the config file format. ");

        OptionGroup ogConfig = new OptionGroup();
        ogConfig.addOption(opConfigFile);

        OptionGroup ogSchema = new OptionGroup();
        ogConfig.addOption(opSchemaFile);

        options.addOptionGroup(ogConfig);
        options.addOptionGroup(ogSchema);

        PosixParser optParser = new PosixParser();
        CommandLine cmdLine = null;
        try {
            cmdLine = optParser.parse(options, args, true);
        } catch (org.apache.commons.cli.ParseException e) {
            logger.error("Error while parsing command-line arguments.");
            System.exit(1);
        }

        Option opt;
        Iterator<?> iter = cmdLine.iterator();
        while (iter.hasNext()) {
            opt = (Option) iter.next();
            if (opt.getOpt().equals("c")) {
                configFile = new File(opt.getValue());
            } else if (opt.getOpt().equals("s")) {
                schemaFile = new File(opt.getValue());
            }
        }

        // ==== Configuration Read and Parse Actions ====
        ConfigParser parser = new ConfigParser(schemaFiles);
        ConfigT config = parser.parseConfig(configFile);

        // Print out configuration file contents
        Formatter formatter = new Formatter(new StringBuilder(), Locale.US);

        // ===== CONFIG: LOCK FILE =====
        File lockFile = new File(config.getLockfile());
        logger.info("SeedScan lock file is '" + lockFile + "'");
        LockFile lock = new LockFile(lockFile);
        if (!lock.acquire()) {
            logger.error("Could not acquire lock.");
            System.exit(1);
        }

        // ===== CONFIG: LOGGING =====
        // MTH: This is now done in log4j.properties file

        // ===== CONFIG: DATABASE =====
        MetricDatabase readDB = new MetricDatabase(config.getDatabase());
        MetricDatabase writeDB = new MetricDatabase(config.getDatabase());
        MetricReader reader = new MetricReader(readDB);
        MetricInjector injector = new MetricInjector(writeDB);

        // ===== CONFIG: SCANS =====
        Hashtable<String, Scan> scans = new Hashtable<String, Scan>();
        if (config.getScans().getScan() == null) {
            logger.error("No scans in configuration.");
            System.exit(1);
        } else {
            for (ScanT scanCfg : config.getScans().getScan()) {
                String name = scanCfg.getName();
                if (scans.containsKey(name)) {
                    logger.error("Duplicate scan name '" + name + "' encountered.");
                    System.exit(1);
                }

                // This should really be handled by jaxb by setting it up in schemas/SeedScanConfig.xsd
                if (scanCfg.getStartDay() == null && scanCfg.getStartDate() == null) {
                    logger.error(
                            "== SeedScan Error: Must set EITHER cfg:start_day -OR- cfg:start_date in config.xml to start Scan!");
                    System.exit(1);
                }

                // Configure this Scan
                Scan scan = new Scan(scanCfg.getName());
                scan.setPathPattern(scanCfg.getPath());
                scan.setDatalessDir(scanCfg.getDatalessDir());
                scan.setEventsDir(scanCfg.getEventsDir());
                scan.setPlotsDir(scanCfg.getPlotsDir());
                scan.setDaysToScan(scanCfg.getDaysToScan().intValue());
                if (scanCfg.getStartDay() != null) {
                    scan.setStartDay(scanCfg.getStartDay().intValue());
                }
                if (scanCfg.getStartDate() != null) {
                    scan.setStartDate(scanCfg.getStartDate().intValue());
                }

                if (scanCfg.getNetworkSubset() != null) {
                    logger.debug("Filter on Network Subset=[{}]", scanCfg.getNetworkSubset());
                    Filter filter = new Filter(false);
                    for (String network : scanCfg.getNetworkSubset().split(",")) {
                        logger.debug("Network =[{}]", network);
                        filter.addFilter(network);
                    }
                    scan.setNetworks(filter);
                }
                if (scanCfg.getStationSubset() != null) {
                    logger.debug("Filter on Station Subset=[{}]", scanCfg.getStationSubset());
                    Filter filter = new Filter(false);
                    for (String station : scanCfg.getStationSubset().split(",")) {
                        logger.debug("Station =[{}]", station);
                        filter.addFilter(station);
                    }
                    scan.setStations(filter);
                }
                if (scanCfg.getLocationSubset() != null) {
                    logger.debug("Filter on Location Subset=[{}]", scanCfg.getLocationSubset());
                    Filter filter = new Filter(false);
                    for (String location : scanCfg.getLocationSubset().split(",")) {
                        logger.debug("Location =[{}]", location);
                        filter.addFilter(location);
                    }
                    scan.setLocations(filter);
                }
                if (scanCfg.getChannelSubset() != null) {
                    logger.debug("Filter on Channel Subset=[{}]", scanCfg.getChannelSubset());
                    Filter filter = new Filter(false);
                    for (String channel : scanCfg.getChannelSubset().split(",")) {
                        logger.debug("Channel =[{}]", channel);
                        filter.addFilter(channel);
                    }
                    scan.setChannels(filter);
                }

                for (MetricT met : scanCfg.getMetrics().getMetric()) {
                    try {
                        Class<?> metricClass = Class.forName(met.getClassName());
                        MetricWrapper wrapper = new MetricWrapper(metricClass);
                        for (ArgumentT arg : met.getArgument()) {
                            wrapper.add(arg.getName(), arg.getValue());
                        }
                        scan.addMetric(wrapper);
                    } catch (ClassNotFoundException ex) {
                        logger.error("No such metric class '" + met.getClassName() + "'");
                        System.exit(1);
                    } catch (InstantiationException ex) {
                        logger.error("Could not dynamically instantiate class '" + met.getClassName() + "'");
                        System.exit(1);
                    } catch (IllegalAccessException ex) {
                        logger.error("Illegal access while loading class '" + met.getClassName() + "'");
                        System.exit(1);
                    } catch (NoSuchFieldException ex) {
                        logger.error("Invalid dynamic argument to Metric subclass '" + met.getClassName() + "'");
                        System.exit(1);
                    }

                }
                scans.put(name, scan);
            }
        }

        // ==== Establish Database Connection ====
        // TODO: State Tracking in the Database
        // - Record scan started in database.
        // - Track our progress as we go so a new process can pick up where
        //   we left off if our process dies.
        // - Mark when each date-station-channel-operation is complete
        //LogDatabaseHandler logDB = new LogDatabaseHandler(configuration.get

        // For each day ((yesterday - scanDepth) to yesterday)
        // scan for these channel files, only process them if
        // they have not yet been scanned, or if changes have
        // occurred to the file since its last scan. Do this for
        // each scan type. Do not re-scan data for each type,
        // launch processes for each scan and use the same data set
        // for each. If we can pipe the data as it is read, do so.
        // If we need to push all of it at once, do these in sequence
        // in order to preserve overall system memory resources.

        Scan scan = null;

        // ==== Perform Scans ====

        scan = scans.get("daily");

        //MTH: This part could/should be moved up higher except that we need to know datalessDir, which,
        //     at this point, is configured on a per scan basis ... so we need to know what scan we're doing
        MetaServer metaServer = null;
        if (config.getMetaserver() != null) {
            if (config.getMetaserver().getUseRemote().equals("yes")
                    || config.getMetaserver().getUseRemote().equals("true")) {
                String remoteServer = config.getMetaserver().getRemoteUri();
                try {
                    metaServer = new MetaServer(new URI(remoteServer));
                } catch (Exception e) {
                    logger.error("caught URI exception:" + e.getMessage());
                }
            } else {
                metaServer = new MetaServer(scan.getDatalessDir());
            }
        } else { // Use local MetaServer
            metaServer = new MetaServer(scan.getDatalessDir());
        }

        List<Station> stations = null;

        if (config.getStationList() == null) { // get StationList from MetaServer
            logger.info("Get StationList from MetaServer");
            stations = metaServer.getStationList();
        } else { // read StationList from config.xml
            logger.info("Read StationList from config.xml");
            List<String> stationList = config.getStationList().getStation();
            if (stationList.size() > 0) {
                stations = new ArrayList<Station>();
                for (String station : stationList) {
                    String[] words = station.split("_");
                    if (words.length != 2) {
                        logger.warn(String.format("stationList: station=[%s] is NOT a valid station --> Skip",
                                station));
                    } else {
                        stations.add(new Station(words[0], words[1]));
                        logger.info("config.xml: Read station:" + station);
                    }
                }
            } else {
                logger.error("Error: No valid stations read from config.xml");
            }
        }

        if (stations == null) {
            logger.error("Found NO stations to scan --> EXITTING SeedScan");
            System.exit(1);
        }

        Thread readerThread = new Thread(reader);
        readerThread.start();
        logger.info("Reader thread started.");

        Thread injectorThread = new Thread(injector);
        injectorThread.start();
        logger.info("Injector thread started.");

        // Loop over scans and hand each one to a ScanManager
        logger.info("Hand scan to ScanManager");
        for (String key : scans.keySet()) {
            scan = scans.get(key);
            logger.info(String.format("Scan=[%s] startDay=%d startDate=%d daysToScan=%d\n", key, scan.getStartDay(),
                    scan.getStartDate(), scan.getDaysToScan()));
            ScanManager scanManager = new ScanManager(reader, injector, stations, scan, metaServer);
        }

        logger.info("ScanManager is [ FINISHED ] --> stop the injector and reader threads");

        try {
            injector.halt();
            logger.info("All stations processed. Waiting for injector thread to finish...");
            synchronized (injectorThread) {
                //injectorThread.wait();
                injectorThread.interrupt();
            }
            logger.info("Injector thread halted.");
        } catch (InterruptedException ex) {
            logger.warn("The injector thread was interrupted while attempting to complete requests.");
        }

        try {
            reader.halt();
            logger.info("All stations processed. Waiting for reader thread to finish...");
            synchronized (readerThread) {
                //readerThread.wait();
                readerThread.interrupt();
            }
            logger.info("Reader thread halted.");
        } catch (InterruptedException ex) {
            logger.warn("The reader thread was interrupted while attempting to complete requests.");
        }

        try {
            lock.release();
        } catch (IOException e) {
            ;
        } finally {
            logger.info("Release seedscan lock and quit metaServer");
            lock = null;
            metaServer.quit();
        }
    } // main()

} // class SeedScan