onl.area51.gfs.grib2.job.GribRetriever.java Source code

Java tutorial

Introduction

Here is the source code for onl.area51.gfs.grib2.job.GribRetriever.java

Source

/*
 * Copyright 2015 peter.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package onl.area51.gfs.grib2.job;

import static java.time.temporal.ChronoField.*;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.temporal.ChronoField;
import java.time.temporal.ChronoUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Stream;
import org.apache.commons.net.ftp.FTPFile;
import uk.trainwatch.io.ftp.FTPClient;
import uk.trainwatch.io.ftp.FTPClientBuilder;

/**
 * Handles the retrieval of GFS GRIB files from NOAA
 * <p>
 * @author peter
 */
public class GribRetriever implements AutoCloseable {

    private static final Logger LOG = Logger.getLogger(GribRetriever.class.getName());

    /**
     * Remote server details
     */
    private static final String SERVER = "ftp.ncep.noaa.gov";
    private static final String DIR = "/pub/data/nccf/com/gfs/prod";

    /**
     * The GRIB product.
     * <p>
     * 0p25 is the 0.25 degree resolution. Could also use 0p50 1p00 or 2p50 for 0.5,1.0 and 2.5 degree resolutions but we'll use the highest one.
     * <p>
     * First arg is the cycle runtime, 00, 06, 12 or 18.
     * <p>
     * Second arg is the forecast hour of product from 000-384
     */
    private static final String PRODUCT = "gfs.t%02dz.pgrb2.0p25.f%03f";

    public static final DateTimeFormatter DIR_FORMATTER;

    static {
        DIR_FORMATTER = new DateTimeFormatterBuilder().parseCaseInsensitive().appendValue(YEAR, 4)
                .appendValue(MONTH_OF_YEAR, 2).appendValue(DAY_OF_MONTH, 2).appendValue(HOUR_OF_DAY).toFormatter();
    }

    private final FTPClient client;

    /**
     * Connect to NOAA
     * <p>
     * @throws IOException
     */
    public GribRetriever() throws IOException {
        // Note: Don't put this in try-resources as it lives longer than the try block
        client = new FTPClientBuilder().logger(LOG::info).build();

        //        try {
        //            LOG.log( Level.INFO, "Connecting to NOAA..." );
        //            client.connect( SERVER );
        //            client.login( "anonymous", "gribuser@" );
        //        }
        //        catch( IOException ex ) {
        //            // Make certain the client is closed correctly
        //            client.close();
        //            throw ex;
        //        }
    }

    /**
     * Select the latest GFS run
     * <p>
     * @throws IOException
     */
    public void selectLatestRun() throws IOException {
        selectRun(null);
    }

    /**
     * Convert a {@link LocalDateTime} to a GFS run time. Specifically this is the date and hour of the day restricted to 0, 7, 12 or 18 hours.
     * <p>
     * @param date date
     * <p>
     * @return date modified to the nearest GFS run (earlier than date) or null if date was null
     */
    public static LocalDateTime toGFSRunTime(LocalDateTime date) {
        if (date == null) {
            return null;
        }

        LocalDateTime dateTime = date.truncatedTo(ChronoUnit.HOURS);

        // Only allow hours 0, 6, 12 & 18
        int h = dateTime.get(ChronoField.HOUR_OF_DAY);
        if (h % 6 == 0) {
            return dateTime;
        }

        return dateTime.withHour((h / 6) * 6);
    }

    /**
     * Select the specified run.
     * <p>
     * @param date The date and hour of the required run
     * <p>
     * @throws java.io.IOException
     */
    public void selectRun(LocalDateTime date) throws IOException {
        LOG.log(Level.INFO, "Retrieving directory listing");
        client.changeWorkingDirectory(DIR);
        Stream<FTPFile> dirs = client.directories().filter(f -> f.getName().startsWith("gfs."));

        if (date == null) {
            // Look for the most recent date
            dirs = dirs.sorted((a, b) -> b.getName().compareToIgnoreCase(a.getName()));
        } else {
            // Look for a specific date
            String filter = "gfs." + toGFSRunTime(date).format(DateTimeFormatter.BASIC_ISO_DATE);
            dirs = dirs.filter(f -> filter.equals(f.getName()));
        }

        @SuppressWarnings("ThrowableInstanceNotThrown")
        FTPFile dir = dirs.findFirst()
                .orElseThrow(() -> new FileNotFoundException("Failed to find remote gfs directory "));

        client.changeWorkingDirectory(dir.getName());
        String pwd = client.printWorkingDirectory();
        LOG.log(Level.INFO, () -> "Now in directory " + pwd);
    }

    public Path retrieveOffset(int offset) throws IOException {
        return retrieveOffset(null, offset, false);
    }

    public Path retrieveOffset(Path dir, int offset) throws IOException {
        return retrieveOffset(dir, offset, false);
    }

    public Path retrieveOffset(int offset, boolean forceRetrive) throws IOException {
        return retrieveOffset(null, offset, forceRetrive);
    }

    public Path retrieveOffset(Path dir, int offset, boolean forceRetrive) throws IOException {
        LOG.log(Level.INFO, () -> "Looking for GFS file for offset " + offset);
        String ending = String.format("z.pgrb2.0p25.f%03d", offset);

        @SuppressWarnings("ThrowableInstanceNotThrown")
        FTPFile remote = client.files().filter(f -> f.getName().startsWith("gfs.t"))
                .filter(f -> f.getName().endsWith(ending)).peek(f -> LOG.warning(f.getName())).findAny()
                .orElseThrow(() -> new FileNotFoundException("Unable to find GFS file for " + offset));

        Path file = dir.resolve(remote.getName());

        if (forceRetrive || client.isPathRetrievable(file, remote)) {
            LOG.log(Level.INFO, () -> "Retrieving " + remote.getSize() + " bytes to " + file);

            try (InputStream is = client.retrieveFileStream(remote)) {
                Files.copy(is, file, StandardCopyOption.REPLACE_EXISTING);
            }

            LOG.log(Level.INFO, () -> "Retrieved " + remote.getSize() + " bytes");
        } else {
            LOG.log(Level.INFO, () -> "Skipping retrieval as local file is newer");
        }

        return file;
    }

    @Override
    public void close() throws IOException {
        client.close();
    }
}