com.boundlessgeo.geoserver.bundle.BundleExporter.java Source code

Java tutorial

Introduction

Here is the source code for com.boundlessgeo.geoserver.bundle.BundleExporter.java

Source

/* (c) 2014 Boundless, http://boundlessgeo.com
 * This code is licensed under the GPL 2.0 license.
 */
package com.boundlessgeo.geoserver.bundle;

import com.boundlessgeo.geoserver.json.JSONObj;
import com.boundlessgeo.geoserver.json.JSONWrapper;
import com.google.common.collect.Maps;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.geoserver.catalog.Catalog;
import org.geoserver.catalog.CatalogBuilder;
import org.geoserver.catalog.CatalogInfo;
import org.geoserver.catalog.CoverageInfo;
import org.geoserver.catalog.CoverageStoreInfo;
import org.geoserver.catalog.DataStoreInfo;
import org.geoserver.catalog.FeatureTypeInfo;
import org.geoserver.catalog.Info;
import org.geoserver.catalog.LayerInfo;
import org.geoserver.catalog.NamespaceInfo;
import org.geoserver.catalog.ResourceInfo;
import org.geoserver.catalog.StoreInfo;
import org.geoserver.catalog.StyleInfo;
import org.geoserver.catalog.WorkspaceInfo;
import org.geoserver.catalog.util.CloseableIterator;
import org.geoserver.config.GeoServerDataDirectory;
import org.geoserver.config.util.XStreamPersister;
import org.geoserver.config.util.XStreamPersisterFactory;
import org.geoserver.data.util.IOUtils;
import org.geoserver.ows.util.OwsUtils;
import org.geoserver.platform.GeoServerResourceLoader;
import org.geotools.data.DataAccessFactory;
import org.geotools.data.DataAccessFactory.Param;
import org.geotools.data.DataUtilities;
import org.geotools.data.FileDataStoreFactorySpi;
import org.geotools.data.property.PropertyDataStoreFactory;
import org.geotools.data.simple.SimpleFeatureSource;
import org.geotools.geometry.jts.Geometries;
import org.geotools.geopkg.FeatureEntry;
import org.geotools.geopkg.GeoPackage;
import org.geotools.geopkg.GeoPkgDataStoreFactory;
import org.geotools.util.logging.Logging;
import org.opengeo.GeoServerInfo;
import org.opengis.feature.simple.SimpleFeatureType;
import org.opengis.feature.type.FeatureType;
import org.opengis.feature.type.GeometryDescriptor;
import org.opengis.filter.Filter;
import org.vfny.geoserver.util.DataStoreUtils;

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Serializable;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.ZipOutputStream;

import static org.geoserver.catalog.Predicates.*;

/**
 * Exports GeoServer config/data bundle.
 */
public class BundleExporter {

    static Logger LOG = Logging.getLogger(BundleExporter.class);

    Catalog catalog;

    Path temp, work;

    GeoServerDataDirectory sourceDataDir;
    GeoServerDataDirectory exportDataDir;

    XStreamPersister xsp;

    ExportOpts options;
    WorkspaceInfo workspace;

    public BundleExporter(Catalog catalog, ExportOpts options) throws IOException {
        this.catalog = catalog;
        this.options = options;
        this.workspace = options.workspace();

        // create a temp directory for the export
        temp = Files.createTempDirectory(null);
        work = temp.resolve("work");
        work.toFile().mkdirs();

        sourceDataDir = new GeoServerDataDirectory(catalog.getResourceLoader());
        exportDataDir = new GeoServerDataDirectory(new GeoServerResourceLoader(work.toFile()));
        //exportDataDir.setConfigFileExtension("json");

        // config serializer
        xsp = new XStreamPersisterFactory().createXMLPersister();
        xsp.setExcludeIds();
        xsp.setReferenceByName(true);
    }

    public Path root() {
        return exportDataDir.get(workspace).dir().toPath();
    }

    public Path work() {
        return work;
    }

    /**
     * Runs the export.
     *
     * @return The path pointing to the root directory of the bundle.
     */
    public Path run() throws Exception {
        //TODO: layer groups

        persistBundleInfo();
        persist(workspace);

        try (CloseableIterator<StoreInfo> sit = catalog.list(StoreInfo.class, equal("workspace", workspace))) {
            while (sit.hasNext()) {
                StoreInfo store = sit.next();

                persist(store);
            }
        }

        return root();
    }

    /**
     * Packages up the export as a zip file.
     *
     * @return The path pointing to the zip file.
     */
    public Path zip() throws IOException {
        Path zip = temp.resolve(options.name() + ".zip");
        try (OutputStream out = new BufferedOutputStream(new FileOutputStream(zip.toFile()));) {
            ZipOutputStream zout = new ZipOutputStream(out);
            try {
                IOUtils.zipDirectory(root().toFile(), zout, null);
            } finally {
                zout.flush();
                zout.close();
                out.flush();
            }
        }
        return zip;
    }

    /**
     * Cleans up the temporary space used by the bundle export.
     */
    public void cleanup() throws IOException {
        FileUtils.deleteDirectory(temp.toFile());
    }

    void persistBundleInfo() throws IOException {
        JSONObj obj = new JSONObj();
        obj.put("name", options.name());

        JSONObj meta = obj.putObject("metadata");
        try {
            GeoServerInfo info = new GeoServerInfo(catalog.getResourceLoader());

            GeoServerInfo.BuildInfo suiteInfo = info.suite();
            meta.putObject("suite").put("version", suiteInfo.version()).put("revision", suiteInfo.revision());

            GeoServerInfo.BuildInfo gsInfo = info.geoserver();
            meta.putObject("geoserver").put("version", suiteInfo.version()).put("revision", suiteInfo.revision());

        } catch (Exception e) {
            throw new IOException(e);
        }

        File wsDir = exportDataDir.get(workspace).dir();
        try (OutputStream out = new BufferedOutputStream(new FileOutputStream(new File(wsDir, "bundle.json")));) {
            JSONWrapper.write(obj, out);
            out.flush();
        }

    }

    void persist(WorkspaceInfo ws) throws IOException {
        persist(ws, exportDataDir.config(ws).file());

        NamespaceInfo ns = catalog.getNamespaceByPrefix(ws.getName());
        if (ns != null) {
            persist(ns, exportDataDir.config(ns).file());
        }
    }

    void persist(StoreInfo s) throws IOException {
        File storeDir = exportDataDir.get(s).dir();

        // create a "data" directory under the workspace
        File wsDataDir = exportDataDir.get(s.getWorkspace()).get("data").dir();

        //TODO: wms store
        File file = null;
        if (s instanceof CoverageStoreInfo) {
            file = exportDataDir.config((CoverageStoreInfo) s).file();
        } else if (s instanceof DataStoreInfo) {
            file = exportDataDir.config((DataStoreInfo) s).file();
        } else {
            file = new File(storeDir, "store.xml");
        }

        if (s instanceof DataStoreInfo) {
            DataStoreInfo ds = (DataStoreInfo) s;

            FileParam dataFile = isFileBased(ds);
            if (dataFile != null && options.bounds() == null) {
                // optimize by copying files over directly
                File storeDataDir = new File(wsDataDir, s.getName());
                storeDataDir.mkdirs();

                copyFilesTo(dataFile.file, storeDataDir, ds);

                // update the store configuration to point to the newly copied files
                File newFileRef = null;
                if (dataFile.file.isDirectory()) {
                    newFileRef = storeDataDir;
                } else {
                    newFileRef = new File(storeDataDir, dataFile.file.getName());
                }

                // TODO: convert back to whatever format the parameter expects
                DataStoreInfo clone = copy(ds, catalog.getFactory().createDataStore(), DataStoreInfo.class);

                //clone.getConnectionParameters().put(dataFile.param.key, "file:%WORKSPACE%/"+newPath.toString());
                clone.getConnectionParameters().put(dataFile.param.key, toWorkspaceRelativePath(newFileRef));

                s = clone;
            } else {
                // copy into a new geopackage
                GeoPackage gpkg = new GeoPackage(new File(wsDataDir, ds.getName() + ".gpkg"));
                try {
                    ingestInto(gpkg, ds);
                } finally {
                    gpkg.close();
                }

                // update the connection parameters
                DataStoreInfo clone = copy(ds, catalog.getFactory().createDataStore(), DataStoreInfo.class);
                clone.setType("GeoPackage");

                Map<String, Serializable> oldParams = clone.getConnectionParameters();
                Map<String, Serializable> params = Maps.newHashMap();

                params.put(GeoPkgDataStoreFactory.DBTYPE.key, "geopkg");
                params.put(GeoPkgDataStoreFactory.DATABASE.key, toWorkspaceRelativePath(gpkg.getFile()));
                params.put(GeoPkgDataStoreFactory.NAMESPACE.key,
                        (Serializable) GeoPkgDataStoreFactory.NAMESPACE.lookUp(oldParams));

                oldParams.clear();
                oldParams.putAll(params);

                s = clone;
            }
        } else if (s instanceof CoverageStoreInfo) {

        }

        persist(s, file);

        try (CloseableIterator<ResourceInfo> rit = catalog.list(ResourceInfo.class, equal("store.id", s.getId()))) {
            while (rit.hasNext()) {
                ResourceInfo resource = rit.next();
                persist(resource);
            }
        }
    }

    String toWorkspaceRelativePath(File newFileRef) {
        Path newPath = root().relativize(newFileRef.toPath());
        return "file:%WORKSPACE%/" + newPath.toString();
    }

    FileParam isFileBased(DataStoreInfo ds) throws IOException {
        DataAccessFactory factory = DataStoreUtils.aquireFactory(ds.getConnectionParameters());
        if (factory == null) {
            factory = DataStoreUtils.aquireFactory(ds.getType());
        }

        if (factory == null) {
            throw new IllegalStateException("Unable to obtain datastore factory for: " + ds);
        }

        if (factory instanceof FileDataStoreFactorySpi) {
            // pull out the file
            FileParam fileParam = findFile(ds, factory);
            if (fileParam != null && fileParam.file.exists()) {
                return fileParam;
            }
        } else if (factory instanceof PropertyDataStoreFactory) {
            File file = (File) PropertyDataStoreFactory.DIRECTORY.lookUp(ds.getConnectionParameters());
            return new FileParam(file, PropertyDataStoreFactory.DIRECTORY);
        }

        // TODO: more heuristics
        return null;
    }

    FileParam findFile(DataStoreInfo ds, DataAccessFactory factory) {
        Param fileParam = null;
        for (Param p : factory.getParametersInfo()) {
            Class<?> type = p.getType();
            if (File.class.isAssignableFrom(type) || URL.class.isAssignableFrom(type)
                    || URI.class.isAssignableFrom(type)) {
                fileParam = p;
                break;
            }
        }

        if (fileParam == null) {
            return null;
        }

        Object obj = ds.getConnectionParameters().get(fileParam.key);
        File file = toFile(obj);

        return file != null ? new FileParam(file, fileParam) : null;
    }

    File toFile(Object obj) {
        if (obj instanceof File) {
            return (File) obj;
        } else if (obj instanceof String) {
            String str = obj.toString();
            if (str.startsWith("file:")) {
                // turn into a url first
                try {
                    return DataUtilities.urlToFile(new URL(str));
                } catch (MalformedURLException e) {
                    // ignore
                }
            }
            return new File(str);
        } else if (obj instanceof URL) {
            return DataUtilities.urlToFile((URL) obj);
        } else if (obj instanceof URI) {
            try {
                return DataUtilities.urlToFile(((URI) obj).toURL());
            } catch (MalformedURLException e) {
                LOG.log(Level.WARNING, "Unable to turn: " + obj + " into file", e);
            }
        }
        return null;
    }

    //    Object to(File file, Class<?> type) {
    //        if (String.class.isAssignableFrom(type)) {
    //            return file.getPath();
    //        }
    //        if (URL.class.isAssignableFrom(type)) {
    //
    //        }
    //        else if (URI.class.isAssignableFrom(type)) {
    //
    //        }
    //        return file;
    //    }

    void copyFilesTo(File dataFile, File dir, DataStoreInfo store) throws IOException {
        List<File> filesToCopy = new ArrayList<>();
        if (dataFile.isDirectory()) {
            // grab all files for feature types in the store
            for (String featureType : nativeFeatureTypeNames(store)) {
                filesToCopy.addAll(filesWithBasename(dataFile, featureType));
            }
        } else {
            // round up all files that have the same base name
            filesToCopy = filesWithBasename(dataFile, FilenameUtils.getBaseName(dataFile.getName()));
        }

        for (File f : filesToCopy) {
            FileUtils.copyFileToDirectory(f, dir);
        }
    }

    void ingestInto(GeoPackage gpkg, DataStoreInfo store) throws IOException {
        try (CloseableIterator<FeatureTypeInfo> it = catalog.list(FeatureTypeInfo.class,
                equal("store.id", store.getId()));) {
            while (it.hasNext()) {
                FeatureTypeInfo ft = it.next();
                FeatureType schema = ft.getFeatureType();
                if (!(schema instanceof SimpleFeatureType)) {
                    LOG.warning("Skipping feature type: " + ft.getName() + ", only simple schema are supported");
                    continue;
                }

                FeatureEntry fe = new FeatureEntry();
                fe.setTableName(ft.getName());
                fe.setIdentifier(ft.getTitle());
                fe.setDescription(ft.getAbstract());
                try {
                    fe.setBounds(ft.boundingBox());
                } catch (Exception e) {
                    throw new IOException(e);
                }
                fe.setSrid(srid(ft.getSRS()));

                GeometryDescriptor geom = schema.getGeometryDescriptor();
                if (geom != null) {
                    fe.setGeometryColumn(geom.getLocalName());
                    fe.setGeometryType(Geometries.getForBinding(
                            (Class<? extends com.vividsolutions.jts.geom.Geometry>) geom.getType().getBinding()));
                }

                gpkg.add(fe, (SimpleFeatureSource) ft.getFeatureSource(null, null), Filter.INCLUDE);
            }
        }
    }

    List<String> nativeFeatureTypeNames(DataStoreInfo store) {
        List<String> names = new ArrayList<>();
        try (CloseableIterator<FeatureTypeInfo> it = catalog.list(FeatureTypeInfo.class,
                equal("store.id", store.getId()));) {
            while (it.hasNext()) {
                names.add(it.next().getNativeName());
            }
        }
        return names;
    }

    List<File> filesWithBasename(File dir, final String basename) {
        return Arrays.asList(dir.listFiles(new FilenameFilter() {
            @Override
            public boolean accept(File dir, String name) {
                return FilenameUtils.getBaseName(name).equals(basename);
            }
        }));
    }

    static Pattern SRID = Pattern.compile(".*:(\\d+)");

    Integer srid(String srs) {
        Matcher m = SRID.matcher(srs);
        if (m.matches()) {
            return Integer.parseInt(m.group(1));
        }
        return null;
    }

    <T extends Info> T copy(T src, T dst, Class<T> type) {
        OwsUtils.copy(src, dst, type);
        OwsUtils.set(dst, "id", src.getId());
        return dst;
    }

    void persist(ResourceInfo r) throws IOException {
        File dir = exportDataDir.get(r).dir();
        dir.mkdirs();

        File file = null;
        if (r instanceof CoverageInfo) {
            file = exportDataDir.config((CoverageInfo) r).file();
        } else if (r instanceof FeatureTypeInfo) {
            file = exportDataDir.config((FeatureTypeInfo) r).file();
        } else {
            file = new File(dir, "resource.xml");
        }

        persist(r, file);

        List<LayerInfo> layers = catalog.getLayers(r);
        if (!layers.isEmpty()) {
            persist(layers.get(0));
        } else {
            LOG.warning("Resource: " + r.getName() + " has no layer");
        }

    }

    void persist(LayerInfo l) throws IOException {
        persist(l.getDefaultStyle());
        for (StyleInfo s : l.getStyles()) {
            persist(s);
        }
        persist(l, exportDataDir.config(l).file());
    }

    void persist(StyleInfo s) throws IOException {
        // grab the stylsheet
        File styleFile = sourceDataDir.style(s).file();

        // if the style is global, convert it to a workpace local one
        if (s.getWorkspace() == null) {
            s = convertToWorkspaceLocal(s);
        }

        File dir = exportDataDir.get(s).dir();
        dir.mkdirs();

        persist(s, exportDataDir.config(s).file());
        FileUtils.copyFileToDirectory(styleFile, dir);

        //TODO: grab all of the icons
    }

    StyleInfo convertToWorkspaceLocal(StyleInfo s) {
        StyleInfo newStyle = catalog.getFactory().createStyle();
        new CatalogBuilder(catalog).updateStyle(newStyle, s);

        newStyle.setWorkspace(options.workspace());
        return newStyle;
    }

    void persist(CatalogInfo info, File file) throws IOException {
        try (OutputStream out = new BufferedOutputStream(new FileOutputStream(file))) {
            xsp.save(info, out);
        }
    }

    static class FileParam {
        final File file;
        final Param param;

        public FileParam(File file, Param param) {
            this.file = file;
            this.param = param;
        }
    }
}