org.schedulesdirect.grabber.Grabber.java Source code

Java tutorial

Introduction

Here is the source code for org.schedulesdirect.grabber.Grabber.java

Source

/*
 *      Copyright 2012-2015 Battams, Derek
 *       
 *       Licensed under the Apache License, Version 2.0 (the "License");
 *       you may not use this file except in compliance with the License.
 *       You may obtain a copy of the License at
 *
 *          http://www.apache.org/licenses/LICENSE-2.0
 *
 *       Unless required by applicable law or agreed to in writing, software
 *       distributed under the License is distributed on an "AS IS" BASIS,
 *       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *       See the License for the specific language governing permissions and
 *       limitations under the License.
 */
package org.schedulesdirect.grabber;

import static org.schedulesdirect.grabber.GrabberReturnCodes.ARGS_PARSE_ERR;
import static org.schedulesdirect.grabber.GrabberReturnCodes.CMD_FAILED_ERR;
import static org.schedulesdirect.grabber.GrabberReturnCodes.NO_CMD_ERR;
import static org.schedulesdirect.grabber.GrabberReturnCodes.OK;
import static org.schedulesdirect.grabber.GrabberReturnCodes.SERVICE_OFFLINE_ERR;

import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.io.Writer;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.text.WordUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.Appender;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.FileAppender;
import org.apache.log4j.Layout;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout;
import org.apache.log4j.SimpleLayout;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.schedulesdirect.api.Config;
import org.schedulesdirect.api.EpgClient;
import org.schedulesdirect.api.Lineup;
import org.schedulesdirect.api.Message;
import org.schedulesdirect.api.NetworkEpgClient;
import org.schedulesdirect.api.RestNouns;
import org.schedulesdirect.api.UserStatus;
import org.schedulesdirect.api.ZipEpgClient;
import org.schedulesdirect.api.exception.InvalidCredentialsException;
import org.schedulesdirect.api.exception.ServiceOfflineException;
import org.schedulesdirect.api.json.IJsonRequestFactory;
import org.schedulesdirect.api.json.DefaultJsonRequest;
import org.schedulesdirect.api.json.JsonRequestFactory;
import org.schedulesdirect.api.utils.AiringUtils;
import org.schedulesdirect.api.utils.JsonResponseUtils;
import org.schedulesdirect.grabber.utils.PathUtils;

import com.beust.jcommander.JCommander;
import com.beust.jcommander.ParameterException;
import com.fasterxml.jackson.core.JsonParseException;

/**
 * An application that will download a Schedules Direct user's lineup data from the servers and generates a zip file in response
 * 
 * <p>This application can be used to generate a local cache file of a user's lineup data.  The generated cache file is suitable as a source for
 * the ZipEpgClient class.  That is, you can download a local file of your data and then use that file in your applications instead of always
 * having to contact the Schedules Direct servers for data.</p>
 * 
 * <p>This app can be used as a simple grabber tool if you just need to download the raw JSON data for your account.</p>
 * 
 * <p>To run this app, be sure to download the standalone jar from the project site then execute:</p>
 * 
 * <code>java -jar sdjson-grabber.x.yyyyyyyy.z.jar</code>
 * 
 * <p>That command alone will dump all of the command line arguments required for operation.</p>
 * 
 * @author Derek Battams &lt;derek@battams.ca&gt;
 *
 */
public final class Grabber {
    static private Log LOG = null;

    /**
     * Location of the saved command line args; saves typing on the command line
     */
    static public final File OPTS_FILE = new File(new File(System.getProperty("user.home")), ".sdjson.properties");
    /**
     * Name of the file holding user data in the zip
     */
    static public final String USER_DATA = "user.txt";

    /**
     * Name of the file holding missing series info ids that need to be retried
     */
    static public final String SERIES_INFO_DATA = "seriesInfo.txt";

    /**
     * Max age of an airing before it's considered expired
     */
    static public final long MAX_AIRING_AGE = 3L * 86400000L; // Ignore airings older than this
    /**
     * Name of the "clean" logger used to display app output without level context
     */
    static public final String LOGGER_APP_DISPLAY = "AppDisplay";
    /**
     * Name of the file containing the logo cache details
     */
    static public final String LOGO_CACHE = "logos.txt";

    /**
     * Has a download task failed?
     */
    static volatile boolean failedTask = false;

    /**
     * Version of the grabber app
     */
    static public final String GRABBER_VERSION = initVersion();

    static private String initVersion() {
        try (InputStream grabberProps = Grabber.class
                .getResourceAsStream("/sdjson-grabber-versioning.properties")) {
            if (grabberProps != null) {
                Properties p = new Properties();
                p.load(grabberProps);
                return p.getProperty("VERSION_DISPLAY");
            }
        } catch (IOException e) {
            return "unknown";
        }
        return "unknown";
    }

    /**
     * Supported actions for this app
     * @author Derek Battams &lt;derek@battams.ca&gt;
     *
     */
    static public enum Action {
        GRAB, LIST, ADD, DELETE, INFO, SEARCH, AUDIT, LISTMSGS, DELMSG, AVAILABLE
    }

    static public final Logger getDisplay() {
        return Logger.getLogger(LOGGER_APP_DISPLAY);
    }

    private Collection<String> stationList = null;
    private Set<String> activeProgIds = new HashSet<String>();
    private JCommander parser;
    private GlobalOptions globalOpts;
    private CommandGrab grabOpts;
    private CommandList listOpts;
    private CommandAdd addOpts;
    private CommandDelete delOpts;
    private CommandSearch searchOpts;
    private CommandInfo infoOpts;
    private CommandAudit auditOpts;
    private CommandListMsgs listMsgsOpts;
    private CommandDeleteMsgs delMsgsOpts;
    private CommandAvailable availOpts;
    private boolean freshZip;
    private long start;
    private boolean logosWarned = false;
    private Action action;
    private IJsonRequestFactory factory;
    private JSONObject logoCache;
    private Set<String> cachedSeriesIds;
    private Set<String> missingSeriesIds;

    private ThreadPoolExecutor pool;

    public Grabber(IJsonRequestFactory factory) {
        this.factory = factory;
    }

    private boolean parseArgs(String[] args) {
        List<String> finalArgs = new ArrayList<String>();
        finalArgs.addAll(Arrays.asList(args));
        try {
            String savedUser = null;
            String savedPwd = null;
            if (OPTS_FILE.exists()) {
                Properties props = new Properties();
                Reader r = new FileReader(OPTS_FILE);
                props.load(r);
                r.close();
                savedUser = props.getProperty("user");
                savedPwd = props.getProperty("password");
            }
            globalOpts = new GlobalOptions(savedUser, savedPwd);
            parser = new JCommander(globalOpts) {
                @Override
                public void usage() {
                    JCommander.getConsole().println(String.format("sdjson-grabber v%s/sdjson-api v%s",
                            GRABBER_VERSION, Config.API_VERSION));
                    super.usage();
                }
            };
            parser.setProgramName("sdjson-grabber");
            grabOpts = new CommandGrab();
            parser.addCommand("grab", grabOpts);
            listOpts = new CommandList();
            parser.addCommand("list", listOpts);
            addOpts = new CommandAdd();
            parser.addCommand("add", addOpts);
            delOpts = new CommandDelete();
            parser.addCommand("delete", delOpts);
            searchOpts = new CommandSearch();
            parser.addCommand("search", searchOpts);
            infoOpts = new CommandInfo();
            parser.addCommand("info", infoOpts);
            auditOpts = new CommandAudit();
            parser.addCommand("audit", auditOpts);
            listMsgsOpts = new CommandListMsgs();
            parser.addCommand("listmsgs", listMsgsOpts);
            delMsgsOpts = new CommandDeleteMsgs();
            parser.addCommand("delmsg", delMsgsOpts);
            availOpts = new CommandAvailable();
            parser.addCommand("available", availOpts);
            parser.parse(finalArgs.toArray(new String[finalArgs.size()]));
            if (globalOpts.isHelp()) {
                parser.usage();
                return false;
            }
            if (LOG == null) {
                Appender a = null;
                File logFile = globalOpts.getLogFile();
                if (logFile != null)
                    a = new FileAppender(new SimpleLayout(), logFile.getAbsolutePath(), false);
                else
                    a = new ConsoleAppender(new SimpleLayout());
                Logger.getRootLogger().addAppender(a);
                Logger.getRootLogger().setLevel(globalOpts.getGrabberLogLvl());
                Logger.getLogger("org.apache.http").setLevel(globalOpts.getHttpLogLvl());
                LOG = LogFactory.getLog(Grabber.class);
                Logger l = Logger.getLogger(LOGGER_APP_DISPLAY);
                l.setAdditivity(false);
                l.setLevel(Level.ALL);
                Layout layout = new PatternLayout("%m");
                File consoleFile = globalOpts.getConsole();
                if (consoleFile != null)
                    a = new FileAppender(layout, consoleFile.getAbsolutePath(), false);
                else
                    a = new ConsoleAppender(layout);
                l.addAppender(a);
            }
        } catch (ParameterException e) {
            System.out.println(e.getMessage());
            String cmd = parser.getParsedCommand();
            if (cmd != null && !globalOpts.isHelp())
                parser.usage(cmd);
            else
                parser.usage();
            return false;
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return true;
    }

    private int deleteMessages(EpgClient clnt, Message[] msgs) throws IOException {
        int deleted = 0;
        for (String id : delMsgsOpts.getIds()) {
            for (Message m : msgs) {
                if (m.getId().equals(id)) {
                    clnt.deleteMessage(m);
                    ++deleted;
                    if (LOG.isInfoEnabled())
                        LOG.info("Deleted message with id " + m.getId());
                    break;
                }
            }
        }
        return deleted;
    }

    private void listAllMessages(EpgClient clnt) throws IOException {
        UserStatus status = clnt.getUserStatus();
        listMessages(clnt, status.getSystemMessages(), "SYSTEM MESSAGES\n===============");
        listMessages(clnt, status.getUserMessages(), "USER MESSAGES\n=============");
    }

    private void listMessages(EpgClient clnt, Message[] msgs, String header) throws IOException {
        Logger l = getDisplay();
        if (msgs != null && msgs.length > 0) {
            l.info(header + "\n");
            SimpleDateFormat fmt = Config.get().getDateTimeFormat();
            for (Message m : msgs) {
                l.info(String.format("%-20s ID: %s%n", fmt.format(m.getDate()), m.getId()));
                l.info(String.format("\t%s%n", WordUtils.wrap(m.getContent(), 78, String.format("%n\t"),
                        globalOpts.getConsole() != null)));
            }
        }
    }

    private void listLineups(EpgClient clnt) throws IOException {
        Logger display = getDisplay();
        display.info(String.format("Available lineups for user '%s'%n", globalOpts.getUsername()));
        display.info(String.format(
                "%-20s Description%n==============================================================================%n",
                "Lineup ID"));
        Lineup[] lineups = clnt.getLineups();
        if (lineups != null && lineups.length > 0)
            for (Lineup l : lineups)
                display.info(String.format("%-20s %s %s%n", l.getUri().substring(l.getUri().lastIndexOf('/') + 1),
                        l.getName(), l.getLocation()));
        else
            display.info("No lineups registered to account!");
    }

    private void listLineupsForZip(EpgClient clnt) throws IOException {
        Logger display = getDisplay();
        display.info(String.format("Available lineups in '%s' for zip '%s'%n", searchOpts.getIsoCountry(),
                searchOpts.getPostalCode()));
        display.info(String.format(
                "%-20s Description%n==============================================================================%n",
                "Headend ID"));
        for (Lineup l : clnt.getLineups(searchOpts.getIsoCountry(), searchOpts.getPostalCode()))
            display.info(String.format("%-20s %s %s%n", l.getUri().substring(l.getUri().lastIndexOf('/') + 1),
                    l.getName(), l.getLocation()));
    }

    private boolean addLineup(NetworkEpgClient clnt) {
        boolean oneFailure = false;
        for (String lineup : addOpts.getIds()) {
            try {
                clnt.registerLineup(EpgClient.getUriPathForLineupId(lineup));
            } catch (IOException e) {
                oneFailure = true;
                LOG.error(
                        String.format("Register lineup command failed for '%s' [msg=%s]", lineup, e.getMessage()));
            }
        }
        if (oneFailure)
            return false;
        LOG.info("Headend(s) added successfully!");
        return true;
    }

    private boolean removeHeadend(NetworkEpgClient clnt) {
        boolean oneFailure = false;
        for (String lineup : delOpts.getIds()) {
            try {
                clnt.unregisterLineup(clnt.getLineupByUriPath(EpgClient.getUriPathForLineupId(lineup)));
            } catch (IOException e) {
                oneFailure = true;
                LOG.error(String.format("Unreigster lineup command failed for '%s' [msg=%s]", lineup,
                        e.getMessage()));
            }
        }
        if (oneFailure)
            return false;
        LOG.info("Headend(s) deleted successfully!");
        return true;
    }

    @SuppressWarnings("unchecked")
    private void buildStationList() {
        File stationFile = grabOpts.getStationFile();
        if (stationFile != null && stationFile.canRead()) {
            stationList = new ArrayList<String>();
            try {
                stationList = (List<String>) FileUtils.readLines(stationFile, ZipEpgClient.ZIP_CHARSET.toString());
            } catch (IOException e) {
                LOG.error("IOError", e);
                stationList.clear();
            }
        }
    }

    private ThreadPoolExecutor createThreadPoolExecutor() {
        return new ThreadPoolExecutor(0, globalOpts.getMaxThreads(), 10, TimeUnit.SECONDS,
                new SynchronousQueue<Runnable>(), new ThreadPoolExecutor.CallerRunsPolicy()) {
            @Override
            protected void afterExecute(Runnable r, Throwable t) {
                //super.afterExecute(r, t);
                if (t != null) {
                    Logger log = Logger.getLogger(r.getClass());
                    log.error("Task failed!", t);
                    if (!(r instanceof LogoTask))
                        failedTask = true;
                }
            }
        };
    }

    private void loadSeriesInfoIds(Path root) throws IOException {
        Files.walkFileTree(root, new FileVisitor<Path>() {
            @Override
            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
                return FileVisitResult.SKIP_SUBTREE;
            }

            @Override
            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                String name = file.getFileName().toString();
                name = name.substring(0, name.lastIndexOf('.'));
                cachedSeriesIds.add(name);
                return FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
                throw exc;
            }

            @Override
            public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
                if (exc != null)
                    throw exc;
                return FileVisitResult.CONTINUE;
            }
        });
    }

    private void loadRetryIds(Path p) {
        try {
            for (String s : Files.readAllLines(p, ZipEpgClient.ZIP_CHARSET))
                missingSeriesIds.add(s.trim());
        } catch (IOException e) {
            LOG.debug("IOError", e);
        }
    }

    private void updateZip(NetworkEpgClient clnt) throws IOException, JSONException, JsonParseException {
        Set<String> completedListings = new HashSet<String>();
        LOG.debug(String.format("Using %d worker threads", globalOpts.getMaxThreads()));
        pool = createThreadPoolExecutor();
        start = System.currentTimeMillis();
        File dest = grabOpts.getTarget();
        cachedSeriesIds = new HashSet<String>();
        boolean rmDest = false;
        if (dest.exists()) {
            ZipEpgClient zipClnt = null;
            try {
                zipClnt = new ZipEpgClient(dest);
                if (!zipClnt.getUserStatus().getLastServerRefresh()
                        .before(clnt.getUserStatus().getLastServerRefresh())) {
                    LOG.info(
                            "Current cache file contains latest data from Schedules Direct server; use --force-download to force a new download from server.");
                    boolean force = grabOpts.isForce();
                    if (!force)
                        return;
                    else
                        LOG.warn("Forcing an update of data with the server due to user request!");
                }
            } catch (Exception e) {
                if (grabOpts.isKeep()) {
                    LOG.error("Existing cache is invalid, keeping by user request!", e);
                    return;
                } else {
                    LOG.warn("Existing cache is invalid, deleting it; use --keep-bad-cache to keep existing cache!",
                            e);
                    rmDest = true;
                }
            } finally {
                if (zipClnt != null)
                    try {
                        zipClnt.close();
                    } catch (IOException e) {
                    }
                if (rmDest && !dest.delete())
                    throw new IOException("Unable to delete " + dest);
            }
        }

        freshZip = !dest.exists();
        try (FileSystem vfs = FileSystems.newFileSystem(new URI(String.format("jar:%s", dest.toURI())),
                Collections.singletonMap("create", "true"))) {
            if (freshZip) {
                Path target = vfs.getPath(ZipEpgClient.ZIP_VER_FILE);
                Files.write(target, Integer.toString(ZipEpgClient.ZIP_VER).getBytes(ZipEpgClient.ZIP_CHARSET));
            }
            ProgramCache progCache = ProgramCache.get(vfs);
            Path lineups = vfs.getPath("lineups.txt");
            Files.deleteIfExists(lineups);
            Path scheds = vfs.getPath("/schedules/");
            if (!Files.isDirectory(scheds))
                Files.createDirectory(scheds);
            Path maps = vfs.getPath("/maps/");
            PathUtils.removeDirectory(maps);
            Files.createDirectory(maps);
            Path progs = vfs.getPath("/programs/");
            if (!Files.isDirectory(progs))
                Files.createDirectory(progs);
            Path logos = vfs.getPath("/logos/");
            if (!Files.isDirectory(logos))
                Files.createDirectory(logos);
            Path md5s = vfs.getPath("/md5s/");
            if (!Files.isDirectory(md5s))
                Files.createDirectory(md5s);
            Path cache = vfs.getPath(LOGO_CACHE);
            if (Files.exists(cache)) {
                String cacheData = new String(Files.readAllBytes(cache), ZipEpgClient.ZIP_CHARSET);
                logoCache = Config.get().getObjectMapper().readValue(cacheData, JSONObject.class);
            } else
                logoCache = new JSONObject();
            Path seriesInfo = vfs.getPath("/seriesInfo/");
            if (!Files.isDirectory(seriesInfo))
                Files.createDirectories(seriesInfo);
            loadSeriesInfoIds(seriesInfo);
            missingSeriesIds = Collections.synchronizedSet(new HashSet<String>());
            loadRetryIds(vfs.getPath(SERIES_INFO_DATA));

            JSONObject resp = Config.get().getObjectMapper().readValue(
                    factory.get(DefaultJsonRequest.Action.GET, RestNouns.LINEUPS, clnt.getHash(),
                            clnt.getUserAgent(), globalOpts.getUrl().toString()).submitForJson(null),
                    JSONObject.class);
            if (!JsonResponseUtils.isErrorResponse(resp))
                Files.write(lineups, resp.toString(3).getBytes(ZipEpgClient.ZIP_CHARSET));
            else
                LOG.error("Received error response when requesting lineup data!");

            for (Lineup l : clnt.getLineups()) {
                buildStationList();
                JSONObject o = Config.get().getObjectMapper()
                        .readValue(
                                factory.get(DefaultJsonRequest.Action.GET, l.getUri(), clnt.getHash(),
                                        clnt.getUserAgent(), globalOpts.getUrl().toString()).submitForJson(null),
                                JSONObject.class);
                Files.write(vfs.getPath("/maps", ZipEpgClient.scrubFileName(String.format("%s.txt", l.getId()))),
                        o.toString(3).getBytes(ZipEpgClient.ZIP_CHARSET));
                JSONArray stations = o.getJSONArray("stations");
                JSONArray ids = new JSONArray();
                for (int i = 0; i < stations.length(); ++i) {
                    JSONObject obj = stations.getJSONObject(i);
                    String sid = obj.getString("stationID");
                    if (stationList != null && !stationList.contains(sid))
                        LOG.debug(String.format("Skipped %s; not listed in station file", sid));
                    else if (completedListings.add(sid)) {
                        ids.put(sid);
                        if (!grabOpts.isNoLogos()) {
                            if (logoCacheInvalid(obj))
                                pool.execute(new LogoTask(obj, vfs, logoCache));
                            else if (LOG.isDebugEnabled())
                                LOG.debug(String.format("Skipped logo for %s; already cached!",
                                        obj.optString("callsign", null)));
                        } else if (!logosWarned) {
                            logosWarned = true;
                            LOG.warn("Logo downloads disabled by user request!");
                        }
                    } else
                        LOG.debug(String.format("Skipped %s; already downloaded.", sid));
                    //pool.setMaximumPoolSize(5); // Processing these new schedules takes all kinds of memory!
                    if (ids.length() == grabOpts.getMaxSchedChunk()) {
                        pool.execute(new ScheduleTask(ids, vfs, clnt, progCache, factory));
                        ids = new JSONArray();
                    }
                }
                if (ids.length() > 0)
                    pool.execute(new ScheduleTask(ids, vfs, clnt, progCache, factory));
            }
            pool.shutdown();
            try {
                LOG.debug("Waiting for SchedLogoExecutor to terminate...");
                if (pool.awaitTermination(15, TimeUnit.MINUTES))
                    LOG.debug("SchedLogoExecutor: Terminated successfully.");
                else {
                    failedTask = true;
                    LOG.warn(
                            "SchedLogoExecutor: Termination timed out; some tasks probably didn't finish properly!");
                }
            } catch (InterruptedException e) {
                failedTask = true;
                LOG.warn(
                        "SchedLogoExecutor: Termination interrupted); some tasks probably didn't finish properly!");
            }
            Files.write(cache, logoCache.toString(3).getBytes(ZipEpgClient.ZIP_CHARSET),
                    StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE, StandardOpenOption.CREATE);
            ScheduleTask.commit(vfs);

            pool = createThreadPoolExecutor();
            //pool.setMaximumPoolSize(5); // Again, we've got memory problems
            String[] dirtyPrograms = progCache.getDirtyIds();
            progCache.markAllClean();
            progCache = null;
            LOG.info(String.format("Identified %d program ids requiring an update!", dirtyPrograms.length));
            Collection<String> progIds = new ArrayList<String>();
            for (String progId : dirtyPrograms) {
                progIds.add(progId);
                if (progIds.size() == grabOpts.getMaxProgChunk()) {
                    pool.execute(new ProgramTask(progIds, vfs, clnt, factory, missingSeriesIds, "programs", null,
                            false));
                    progIds.clear();
                }
            }
            if (progIds.size() > 0)
                pool.execute(
                        new ProgramTask(progIds, vfs, clnt, factory, missingSeriesIds, "programs", null, false));
            pool.shutdown();
            try {
                LOG.debug("Waiting for ProgramExecutor to terminate...");
                if (pool.awaitTermination(15, TimeUnit.MINUTES)) {
                    LOG.debug("ProgramExecutor: Terminated successfully.");
                    Iterator<String> itr = missingSeriesIds.iterator();
                    while (itr.hasNext()) {
                        String id = itr.next();
                        if (cachedSeriesIds.contains(id))
                            itr.remove();
                    }
                    if (missingSeriesIds.size() > 0) {
                        LOG.info(String.format("Grabbing %d series info programs!", missingSeriesIds.size()));
                        Set<String> retrySet = new HashSet<>();
                        try {
                            new ProgramTask(missingSeriesIds, vfs, clnt, factory, missingSeriesIds, "seriesInfo",
                                    retrySet, true).run();
                        } catch (RuntimeException e) {
                            LOG.error("SeriesInfo task failed!", e);
                            Grabber.failedTask = true;
                        }
                        Path seriesInfoData = vfs.getPath(SERIES_INFO_DATA);
                        if (retrySet.size() > 0) {
                            StringBuilder sb = new StringBuilder();
                            for (String id : retrySet)
                                sb.append(id + "\n");
                            Files.write(seriesInfoData, sb.toString().getBytes(ZipEpgClient.ZIP_CHARSET),
                                    StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING,
                                    StandardOpenOption.CREATE);
                        } else if (Files.exists(seriesInfoData))
                            Files.delete(seriesInfoData);
                    }
                } else {
                    failedTask = true;
                    LOG.warn("ProgramExecutor: Termination timed out; some tasks probably didn't finish properly!");
                }
            } catch (InterruptedException e) {
                failedTask = true;
                LOG.warn("ProgramExecutor: Termination interrupted); some tasks probably didn't finish properly!");
            }

            String userData = clnt.getUserStatus().toJson();
            if (failedTask) {
                LOG.error("One or more tasks failed!  Resetting last data refresh timestamp to zero.");
                SimpleDateFormat fmt = Config.get().getDateTimeFormat();
                String exp = fmt.format(new Date(0L));
                JSONObject o = Config.get().getObjectMapper().readValue(userData, JSONObject.class);
                o.put("lastDataUpdate", exp);
                userData = o.toString(2);
            }
            Path p = vfs.getPath(USER_DATA);
            Files.write(p, userData.getBytes(ZipEpgClient.ZIP_CHARSET), StandardOpenOption.WRITE,
                    StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE);
            removeIgnoredStations(vfs);
        } catch (URISyntaxException e1) {
            throw new RuntimeException(e1);
        } finally {
            Runtime rt = Runtime.getRuntime();
            LOG.info(String.format("MemStats:%n\tFREE: %s%n\tUSED: %s%n\t MAX: %s",
                    FileUtils.byteCountToDisplaySize(rt.freeMemory()),
                    FileUtils.byteCountToDisplaySize(rt.totalMemory()),
                    FileUtils.byteCountToDisplaySize(rt.maxMemory())));
        }
    }

    private boolean logoCacheInvalid(JSONObject station) throws JSONException, IOException {
        JSONObject logo = station.optJSONObject("logo");
        if (logo != null) {
            String callsign = station.getString("callsign");
            String cached = logoCache.optString(callsign, null);
            if (cached != null)
                return !cached.equals(logo.optString("md5"));
        }
        return true;
    }

    private void removeExpiredSchedules(FileSystem vfs) throws IOException, JSONException {
        final int[] i = new int[] { 0 };
        final Path root = vfs.getPath("schedules");
        Files.walkFileTree(root, new FileVisitor<Path>() {

            @Override
            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
                return !Files.isSameFile(dir, root) ? FileVisitResult.SKIP_SUBTREE : FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                try (InputStream ins = Files.newInputStream(file)) {
                    JSONArray sched = Config.get().getObjectMapper()
                            .readValue(IOUtils.toString(ins, ZipEpgClient.ZIP_CHARSET.toString()), JSONObject.class)
                            .getJSONArray("programs");
                    if (isScheduleExpired(sched)) {
                        Files.delete(file);
                        ++i[0];
                    }
                }
                return FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
                return FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
                return FileVisitResult.CONTINUE;
            }

        });
        LOG.info(String.format("Removed %d expired schedule(s).", i[0]));
    }

    private boolean isScheduleExpired(JSONArray sched) throws JSONException {
        boolean rc = true;
        Date expiry = new Date(System.currentTimeMillis() - (3L * 86400000L));
        for (int i = 0; i < sched.length(); ++i) {
            JSONObject air = sched.getJSONObject(i);
            if (!AiringUtils.getEndDate(air).before(expiry)) {
                rc = false;
                activeProgIds.add(air.getString("programID"));
            }
        }
        return rc;
    }

    private void removeUnusedPrograms(FileSystem vfs) throws IOException {
        final int[] i = new int[] { 0 };
        final Path root = vfs.getPath("programs");
        Files.walkFileTree(root, new FileVisitor<Path>() {

            @Override
            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
                return !Files.isSameFile(root, dir) ? FileVisitResult.SKIP_SUBTREE : FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                String id = file.getName(file.getNameCount() - 1).toString();
                id = id.substring(0, id.indexOf('.'));
                if (!activeProgIds.contains(id)) {
                    if (LOG.isDebugEnabled())
                        LOG.debug(String.format("CacheCleaner: Unused '%s'", id));
                    Files.delete(file);
                    ++i[0];
                }
                return FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
                return FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
                return FileVisitResult.CONTINUE;
            }

        });
        LOG.info(String.format("Removed %d unused program(s).", i[0]));
    }

    private void removeIgnoredStations(FileSystem vfs) throws IOException {
        final int[] i = new int[] { 0 };
        final Path root = vfs.getPath("schedules");
        Files.walkFileTree(root, new FileVisitor<Path>() {

            @Override
            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
                return !Files.isSameFile(root, dir) ? FileVisitResult.SKIP_SUBTREE : FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                String id = file.getName(file.getNameCount() - 1).toString();
                id = id.substring(0, id.indexOf('.'));
                if (stationList != null && !stationList.contains(id)) {
                    if (LOG.isDebugEnabled())
                        LOG.debug(String.format("CacheCleaner: Remove '%s'", id));
                    Files.delete(file);
                    ++i[0];
                }
                return FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
                return FileVisitResult.CONTINUE;
            }

            @Override
            public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
                return FileVisitResult.CONTINUE;
            }

        });
        LOG.info(String.format("Removed %d ignored station(s).", i[0]));
    }

    /**
     * Execute the grabber app
     * @param args The command line args
     * @throws IOException Thrown on any unexpected IO error
     * @throws InvalidCredentialsException Thrown if the login attempt to Schedules Direct failed
     * @return {@link GrabberReturnCodes#OK OK} on success, one of the other constants in the interface otherwise; typically one would return this value back to the OS level caller
     * @see GrabberReturnCodes
     */
    public int execute(String[] args) throws IOException, InvalidCredentialsException {
        if (!parseArgs(args))
            return ARGS_PARSE_ERR;
        else if (parser.getParsedCommand() == null) {
            parser.usage();
            return NO_CMD_ERR;
        }
        NetworkEpgClient clnt = null;
        try {
            if (!parser.getParsedCommand().equals("audit")) {
                try {
                    clnt = new NetworkEpgClient(globalOpts.getUsername(), globalOpts.getPassword(),
                            globalOpts.getUserAgent(), globalOpts.getUrl().toString(), true, factory);
                    LOG.debug(String.format("Client details: %s", clnt.getUserAgent()));
                } catch (ServiceOfflineException e) {
                    LOG.error("Web service is offline!  Please try again later.");
                    return SERVICE_OFFLINE_ERR;
                }
            }
            action = Action.valueOf(parser.getParsedCommand().toUpperCase());
            int rc = CMD_FAILED_ERR;
            switch (action) {
            case LIST:
                if (!listOpts.isHelp()) {
                    listLineups(clnt);
                    rc = OK;
                } else
                    parser.usage(action.toString().toLowerCase());
                break;
            case GRAB:
                if (!grabOpts.isHelp()) {
                    updateZip(clnt);
                    rc = OK;
                } else
                    parser.usage(action.toString().toLowerCase());
                break;
            case ADD:
                if (!addOpts.isHelp())
                    addLineup(clnt);
                else
                    parser.usage(action.toString().toLowerCase());
                break;
            case DELETE:
                if (!delOpts.isHelp())
                    rc = removeHeadend(clnt) ? OK : CMD_FAILED_ERR;
                else
                    parser.usage(action.toString().toLowerCase());
                break;
            case INFO:
                if (!infoOpts.isHelp()) {
                    dumpAccountInfo(clnt);
                    rc = OK;
                } else
                    parser.usage(action.toString().toLowerCase());
                break;
            case SEARCH:
                if (!searchOpts.isHelp()) {
                    listLineupsForZip(clnt);
                    rc = OK;
                } else
                    parser.usage(action.toString().toLowerCase());
                break;
            case AUDIT:
                if (!auditOpts.isHelp()) {
                    Auditor a = new Auditor(auditOpts);
                    a.run();
                    if (!a.isFailed())
                        rc = OK;
                } else
                    parser.usage(action.toString().toLowerCase());
                break;
            case LISTMSGS:
                if (!listMsgsOpts.isHelp()) {
                    listAllMessages(clnt);
                    rc = OK;
                } else
                    parser.usage(action.toString().toLowerCase());
                break;
            case DELMSG:
                if (!delMsgsOpts.isHelp()) {
                    if (deleteMessages(clnt, clnt.getUserStatus().getSystemMessages()) < delMsgsOpts.getIds()
                            .size())
                        deleteMessages(clnt, clnt.getUserStatus().getUserMessages());
                    rc = OK;
                } else
                    parser.usage(action.toString().toLowerCase());
                break;
            case AVAILABLE:
                if (!availOpts.isHelp()) {
                    String type = availOpts.getType();
                    if (type == null)
                        listAvailableThings(clnt);
                    else
                        listAvailableThings(clnt, type); // TODO: change
                    rc = OK;
                } else
                    parser.usage(action.toString().toLowerCase());
                break;
            }
            return rc;
        } catch (ParameterException e) {
            System.out.println(e.getMessage());
            parser.usage();
            return ARGS_PARSE_ERR;
        } catch (JSONException e) {
            throw new RuntimeException(e);
        } finally {
            if (clnt != null)
                clnt.close();
            if (grabOpts.getTarget().exists() && action == Action.GRAB) {
                FileSystem target;
                try {
                    target = FileSystems.newFileSystem(
                            new URI(String.format("jar:%s", grabOpts.getTarget().toURI())),
                            Collections.<String, Object>emptyMap());
                } catch (URISyntaxException e1) {
                    throw new RuntimeException(e1);
                }
                if (action == Action.GRAB && !grabOpts.isHelp() && grabOpts.isPurge() && !freshZip) {
                    LOG.warn("Performing a cache cleanup, this will take a few minutes!");
                    try {
                        removeExpiredSchedules(target);
                    } catch (JSONException e) {
                        throw new IOException(e);
                    }
                    removeUnusedPrograms(target);
                }
                target.close();
                if (action == Action.GRAB && !grabOpts.isHelp())
                    LOG.info(String.format("Created '%s' successfully! [%dms]", target,
                            System.currentTimeMillis() - start));
            }
            if (globalOpts.isSaveCreds()) {
                String user = globalOpts.getUsername();
                String pwd = globalOpts.getPassword();
                if (user != null && user.length() > 0 && pwd != null && pwd.length() > 0) {
                    Properties props = new Properties();
                    props.setProperty("user", globalOpts.getUsername());
                    props.setProperty("password", globalOpts.getPassword());
                    Writer w = new FileWriter(OPTS_FILE);
                    props.store(w, "Generated by sdjson-grabber");
                    w.close();
                    LOG.info(String.format("Credentials saved for future use in %s!", OPTS_FILE.getAbsoluteFile()));
                }
            }
        }
    }

    private void listAvailableThings(NetworkEpgClient clnt) throws IOException {
        getDisplay().info(clnt.getAvailableTypes());
    }

    private void listAvailableThings(NetworkEpgClient clnt, String type) throws IOException {
        type = type.toLowerCase();
        String json = clnt.getAvailableThings(type);
        if ("countries".equals(type))
            getDisplay().info(formatCountries(json));
        else {
            LOG.warn(String.format("Type not supported; dumping raw response [%s]", type));
            getDisplay().info(json);
        }
    }

    private String formatCountries(String json) throws IOException {
        JSONObject resp = Config.get().getObjectMapper().readValue(json, JSONObject.class);
        String fmt = "%-15s %15s %3s %-40s%n";
        getDisplay().info("Use 'ISO' value when adding lineups to your account.");
        getDisplay().info(String.format(fmt, "LOCATION", "", "ISO", "EXAMPLE"));
        getDisplay().info(String.format("%s%n", StringUtils.repeat('=', 76)));
        getDisplay().info(String.format("%s%n", "North America"));
        getDisplay().info(String.format(fmt, "", "United States", "USA", "12345"));
        getDisplay().info(String.format(fmt, "", "Canada", "CAN", "A0A0A0"));
        return resp.toString(3);
    }

    private void dumpAccountInfo(NetworkEpgClient clnt) throws IOException {
        getDisplay().info(String.format("%s%n", clnt.getUserStatus().toJson().trim()));
    }

    public static void main(String[] args) throws Exception {
        int rc = new Grabber(new JsonRequestFactory()).execute(args);
        if (rc != OK)
            System.exit(rc);
    }
}