gov.osti.services.Metadata.java Source code

Java tutorial

Introduction

Here is the source code for gov.osti.services.Metadata.java

Source

package gov.osti.services;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.annotation.JsonFilter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.PropertyNamingStrategy;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.ser.FilterProvider;
import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;

import gov.osti.connectors.ConnectorFactory;
import gov.osti.connectors.BitBucket;
import gov.osti.connectors.GitHub;
import gov.osti.connectors.GitLab;
import gov.osti.connectors.HttpUtil;
import gov.osti.connectors.SourceForge;
import gov.osti.connectors.api.GitLabAPI;
import gov.osti.connectors.gitlab.Commit;
import gov.osti.connectors.gitlab.GitLabFile;
import gov.osti.doi.DataCite;
import gov.osti.entity.Agent;
import gov.osti.entity.Contributor;
import gov.osti.entity.MetadataSnapshot;
import gov.osti.entity.DOECodeMetadata;
import gov.osti.entity.DOECodeMetadata.Accessibility;
import gov.osti.entity.DOECodeMetadata.Status;
import gov.osti.entity.Developer;
import gov.osti.entity.DoiReservation;
import gov.osti.entity.ResearchOrganization;
import gov.osti.entity.Site;
import gov.osti.entity.SponsoringOrganization;
import gov.osti.entity.User;
import gov.osti.indexer.AgentSerializer;
import gov.osti.listeners.DoeServletContextListener;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
import java.lang.reflect.InvocationTargetException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;

import javax.persistence.EntityManager;
import javax.persistence.LockModeType;
import javax.persistence.LockTimeoutException;
import javax.persistence.PessimisticLockException;
import javax.persistence.TypedQuery;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Expression;
import javax.persistence.criteria.ParameterExpression;
import javax.persistence.criteria.Root;
import javax.persistence.criteria.Predicate;
import javax.servlet.ServletContext;
import javax.validation.ConstraintViolation;
import javax.validation.Validator;
import javax.validation.ValidatorFactory;
import javax.validation.ValidationException;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.core.Context;
import javax.ws.rs.Consumes;
import javax.ws.rs.Produces;
import javax.ws.rs.GET;
import javax.ws.rs.InternalServerErrorException;
import javax.ws.rs.NotFoundException;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.commons.beanutils.BeanUtilsBean;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.mail.EmailException;
import org.apache.commons.mail.HtmlEmail;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.entity.mime.HttpMultipartMode;
import org.apache.http.entity.mime.MultipartEntityBuilder;
import org.apache.http.entity.mime.content.FileBody;
import org.apache.http.entity.mime.content.StringBody;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.apache.shiro.SecurityUtils;
import org.apache.shiro.authz.annotation.RequiresAuthentication;
import org.apache.shiro.authz.annotation.RequiresRoles;
import org.apache.shiro.subject.Subject;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataParam;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import gov.osti.connectors.gitlab.Project;
import gov.osti.entity.RelatedIdentifier;
import java.io.FileInputStream;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.TimeZone;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.codec.binary.Base64InputStream;
import static java.nio.file.StandardCopyOption.*;

/**
 * REST Web Service for Metadata.
 *
 * endpoints:
 *
 * GET
 * metadata/{codeId} - retrieve JSON for record if owner/administrator, optionally in various formats
 * metadata/autopopulate?repo={url} - attempt an auto-populate Connector call for
 * indicated URL, optionally in YAML format
 *
 * POST
 * metadata - send JSON for persisting to the storage layer
 * metadata/submit - send JSON for posting to both ELINK and persistence layer
 * metadata/yaml - send JSON, get YAML back
 *
 * @author ensornl
 */
@Path("metadata")
public class Metadata {
    // inject a Context
    @Context
    ServletContext context;

    // logger instance
    private static final Logger log = LoggerFactory.getLogger(Metadata.class);
    private static ConnectorFactory factory;

    // SMTP email host name
    private static final String EMAIL_HOST = DoeServletContextListener.getConfigurationProperty("email.host");
    // EMAIL send-from account name
    private static final String EMAIL_FROM = DoeServletContextListener.getConfigurationProperty("email.from");
    // EMAIL address to send to for SUBMISSION/ANNOUNCE
    private static final String EMAIL_SUBMISSION = DoeServletContextListener
            .getConfigurationProperty("email.notification");
    // URL to indexer services, if configured
    private static String INDEX_URL = DoeServletContextListener.getConfigurationProperty("index.url");
    // absolute filesystem location to store uploaded files, if any
    private static String FILE_UPLOADS = DoeServletContextListener.getConfigurationProperty("file.uploads");
    // absolute filesystem location to store uploaded container images, if any
    private static String CONTAINER_UPLOADS = DoeServletContextListener.getConfigurationProperty("file.containers");
    // absolute filesystem location to store uploaded container images, if any
    private static String CONTAINER_UPLOADS_APPROVED = DoeServletContextListener
            .getConfigurationProperty("file.containers.approved");
    // API path to archiver services if available
    private static String ARCHIVER_URL = DoeServletContextListener.getConfigurationProperty("archiver.url");
    // get the SITE URL base for applications
    private static String SITE_URL = DoeServletContextListener.getConfigurationProperty("site.url");
    // get the SITE URL base for applications
    private static String PM_NAME = DoeServletContextListener.getConfigurationProperty("project.manager.name");
    // get the SITE URL base for applications
    private static String PM_EMAIL = DoeServletContextListener.getConfigurationProperty("project.manager.email");

    // set pattern for DOI normalization
    private static final Pattern DOI_TRIM_PATTERN = Pattern.compile("(10.\\d{4,9}\\/[-._;()\\/:A-Za-z0-9]+)$");
    private static final Pattern URL_TRIM_PATTERN = Pattern.compile("^(.*)(?<!\\/)\\/?$");

    // create and start a ConnectorFactory for use by "autopopulate" service
    static {
        try {
            factory = ConnectorFactory.getInstance().add(new GitHub()).add(new SourceForge()).add(new BitBucket())
                    .add(new GitLab()).build();
        } catch (IOException e) {
            log.warn("Configuration failure: " + e.getMessage());
        }
    }

    /**
     * Creates a new instance of MetadataResource
     */
    public Metadata() {
    }

    /**
     * Implement a simple JSON filter to remove named properties.
     */
    @JsonFilter("filter properties by name")
    class PropertyFilterMixIn {
    }

    // filter out certain attribute names
    protected final static String[] ignoreProperties = { "recipientName", "recipient_name", "recipientEmail",
            "recipient_email", "recipientPhone", "recipient_phone", "owner", "workflowStatus", "workflow_status",
            "accessLimitations", "access_limitations", "disclaimers", "siteOwnershipCode", "site_ownership_code" };
    protected static FilterProvider filter = new SimpleFilterProvider().addFilter("filter properties by name",
            SimpleBeanPropertyFilter.serializeAllExcept(ignoreProperties));

    // ObjectMapper instance for yaml response
    protected static final ObjectMapper YAML_MAPPER = new ObjectMapper(new YAMLFactory())
            .setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE)
            .setSerializationInclusion(JsonInclude.Include.NON_NULL)
            .addMixIn(Object.class, PropertyFilterMixIn.class).setTimeZone(TimeZone.getDefault());
    // ObjectMapper instance for metadata interchange
    private static final ObjectMapper mapper = new ObjectMapper()
            .setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE)
            .setSerializationInclusion(JsonInclude.Include.NON_NULL).setTimeZone(TimeZone.getDefault());
    // ObjectMapper specifically for indexing purposes
    protected static final ObjectMapper index_mapper = new ObjectMapper()
            .setSerializationInclusion(JsonInclude.Include.NON_NULL).setTimeZone(TimeZone.getDefault());
    static {
        // customized serializer module for Agent names consolidation
        SimpleModule module = new SimpleModule();
        module.addSerializer(Agent.class, new AgentSerializer());
        index_mapper.registerModule(module);
    }

    /**
     * Obtain a reserved DOI value if possible.
     *
     * @return a DoiReservation if successful, or null if not
     */
    private static DoiReservation getReservedDoi() {
        EntityManager em = DoeServletContextListener.createEntityManager();
        // set a LOCK TIMEOUT to prevent collision
        em.setProperty("javax.persistence.lock.timeout", 5000);

        try {
            em.getTransaction().begin();

            DoiReservation reservation = em.find(DoiReservation.class, DoiReservation.TYPE,
                    LockModeType.PESSIMISTIC_WRITE);

            if (null == reservation)
                reservation = new DoiReservation();

            reservation.reserve();

            em.merge(reservation);

            em.getTransaction().commit();

            // send it back
            return reservation;
        } catch (PessimisticLockException | LockTimeoutException e) {
            log.warn("DOI Reservation, unable to obtain lock.", e);
            return null;
        } finally {
            em.close();
        }
    }

    /**
     * Acquire a unique DOI reservation value.  Requires authentication.
     *
     * Response Code:
     * 200 - JSON contains "doi" element with a new reserved DOI value
     * 500 - a parser or other unexpected error occurred
     *
     * @throws IOException on JSON parsing errors
     * @return JSON containing a new reserved DOI value.
     */
    @GET
    @Path("reservedoi")
    @Produces(MediaType.APPLICATION_JSON)
    @RequiresAuthentication
    public Response reserveDoi() throws IOException {
        // attempt to reserve a DOI
        DoiReservation reservation = getReservedDoi();

        // if we got a reservation, send it back; otherwise, show a failure
        return (null == reservation)
                ? ErrorResponse.internalServerError("DOI reservation processing failed.").build()
                : Response.ok().entity(mapper.writeValueAsString(reservation)).build();
    }

    /**
     * Look up a record for EDITING, checks authentication and ownership prior
     * to succeeding.
     *
     * Ownership is defined as:  owner and user email match, OR user's roles
     * include the SITE OWNERSHIP CODE of the record, OR user has the "OSTI"
     * special administrative role.
     * Result Codes:
     * 200 - OK, with JSON containing the metadata information
     * 400 - you didn't specify a CODE ID
     * 401 - authentication required
     * 403 - forbidden, logged in user does not have permission to this metadata
     * 404 - requested metadata is not on file
     *
     * @param codeId the CODE ID to look up
     * @param format optional; "yaml" or "xml", default is JSON unless specified
     * @return a Response containing JSON if successful
     */
    @GET
    @Path("{codeId}")
    @Produces({ MediaType.APPLICATION_JSON, "text/yaml", MediaType.APPLICATION_XML })
    @RequiresAuthentication
    @SuppressWarnings("ConvertToStringSwitch")
    public Response getSingleRecord(@PathParam("codeId") Long codeId, @QueryParam("format") String format) {
        EntityManager em = DoeServletContextListener.createEntityManager();
        Subject subject = SecurityUtils.getSubject();
        User user = (User) subject.getPrincipal();

        // no CODE ID?  Bad request.
        if (null == codeId)
            return ErrorResponse.badRequest("Missing code ID.").build();

        DOECodeMetadata md = em.find(DOECodeMetadata.class, codeId);

        // no metadata?  404
        if (null == md)
            return ErrorResponse.notFound("Code ID not on file.").build();

        // do you have permissions to get this?
        if (!user.getEmail().equals(md.getOwner()) && !user.hasRole("OSTI")
                && !user.hasRole(md.getSiteOwnershipCode()))
            return ErrorResponse.forbidden("Permission denied.").build();

        // if YAML is requested, return that; otherwise, default to JSON
        try {
            if ("yaml".equals(format)) {
                // return the YAML (excluding filtered data)
                return Response.ok().header("Content-Type", "text/yaml")
                        .header("Content-Disposition", "attachment; filename = \"metadata.yml\"")
                        .entity(YAML_MAPPER.writer(filter).writeValueAsString(md)).build();
            } else if ("xml".equals(format)) {
                return Response.ok().header("Content-Type", MediaType.APPLICATION_XML).entity(HttpUtil.writeXml(md))
                        .build();
            } else {
                // send back the JSON
                return Response.ok().header("Content-Type", MediaType.APPLICATION_JSON)
                        .entity(mapper.createObjectNode().putPOJO("metadata", md.toJson()).toString()).build();
            }
        } catch (IOException e) {
            log.warn("JSON Output Error", e);
            return ErrorResponse.internalServerError("Unable to process request.").build();
        }
    }

    /**
     * Intended to be a List of retrieved Metadata records/projects.
     */
    private class RecordsList {
        // the records
        private List<DOECodeMetadata> records;
        // a total count of a matched query
        private long total;
        // the starting index (0-based)
        private int start;

        RecordsList(List<DOECodeMetadata> records) {
            this.records = records;
        }

        /**
         * Acquire the list of records (a single page of results).
         *
         * @return a List of DOECodeMetadata Objects
         */
        public List<DOECodeMetadata> getRecords() {
            return records;
        }

        /**
         * Set the current page of results.
         *
         * @param records a List of DOECodeMetadata Objects for this page
         */
        public void setRecords(List<DOECodeMetadata> records) {
            this.records = records;
        }

        /**
         * Set a TOTAL count of matches for this search/list.
         *
         * @param count the count to set
         */
        public void setTotal(long count) {
            total = count;
        }

        /**
         * Get the TOTAL number of matching rows.
         *
         * @return the total count matched
         */
        public long getTotal() {
            return total;
        }

        /**
         * Set the starting index/offset number.
         *
         * @param start the starting index or offset (0 based)
         */
        public void setStart(int start) {
            this.start = start;
        }

        /**
         * Get the starting index number, based on 0.
         *
         * @return the starting index number of the records
         */
        public int getStart() {
            return this.start;
        }

        /**
         * Get the number of rows on the current "page" of results.
         *
         * @return the number of rows in the current list/page of results.
         */
        public int size() {
            return (null == records) ? 0 : records.size();
        }
    }

    /**
     * Acquire a listing of all records by OWNER.
     *
     * @param rows the number of rows desired (if present)
     * @param start the starting row number (from 0)
     * @return the Metadata information in the desired format
     * @throws JsonProcessingException
     */
    @GET
    @Path("/projects")
    @Produces(MediaType.APPLICATION_JSON)
    @RequiresAuthentication
    public Response listProjects(@QueryParam("rows") int rows, @QueryParam("start") int start)
            throws JsonProcessingException {
        EntityManager em = DoeServletContextListener.createEntityManager();

        // get the security user in context
        Subject subject = SecurityUtils.getSubject();
        User user = (User) subject.getPrincipal();

        try {
            Set<String> roles = user.getRoles();
            String rolecode = (null == roles) ? "" : (roles.isEmpty()) ? "" : roles.iterator().next();

            TypedQuery<DOECodeMetadata> query;
            // admins see ALL PROJECTS
            if ("OSTI".equals(rolecode)) {
                query = em.createQuery("SELECT md FROM DOECodeMetadata md", DOECodeMetadata.class);
            } else if (StringUtils.isNotEmpty(rolecode)) {
                // if you have another ROLE, it is assumed to be a SITE ADMIN; see all those records
                query = em.createQuery("SELECT md FROM DOECodeMetadata md WHERE md.siteOwnershipCode = :site",
                        DOECodeMetadata.class).setParameter("site", rolecode);
            } else {
                // no roles, you see only YOUR OWN projects
                query = em.createQuery("SELECT md FROM DOECodeMetadata md WHERE md.owner = lower(:owner)",
                        DOECodeMetadata.class).setParameter("owner", user.getEmail());
            }

            // if rows specified, and greater than 100, cap it there
            rows = (rows > 100) ? 100 : rows;

            // if pagination elements are present, set them on the query
            if (0 != rows)
                query.setMaxResults(rows);
            if (0 != start)
                query.setFirstResult(start);

            // get a List of records
            RecordsList records = new RecordsList(query.getResultList());
            records.setStart(start);
            ObjectNode recordsObject = mapper.valueToTree(records);

            // lookup previous Snapshot status info for each item
            TypedQuery<MetadataSnapshot> querySnapshot = em
                    .createNamedQuery("MetadataSnapshot.findByCodeIdLastNotStatus", MetadataSnapshot.class)
                    .setParameter("status", DOECodeMetadata.Status.Approved);

            // lookup system Snapshot status info for each item
            TypedQuery<MetadataSnapshot> querySystemSnapshot = em
                    .createNamedQuery("MetadataSnapshot.findByCodeIdAsSystemStatus", MetadataSnapshot.class)
                    .setParameter("status", DOECodeMetadata.Status.Approved);

            JsonNode recordNode = recordsObject.get("records");
            if (recordNode.isArray()) {
                int rowCount = 0;
                for (JsonNode objNode : recordNode) {
                    rowCount++;

                    // skip non-approved records
                    String currentStatus = objNode.get("workflow_status").asText();
                    if (!currentStatus.equalsIgnoreCase("Approved"))
                        continue;

                    // get code_id to find Snapshot
                    long codeId = objNode.get("code_id").asLong();
                    querySnapshot.setParameter("codeId", codeId);
                    querySystemSnapshot.setParameter("codeId", codeId);

                    String lastApprovalFor = "";
                    List<MetadataSnapshot> results = querySnapshot.setMaxResults(1).getResultList();
                    for (MetadataSnapshot ms : results) {
                        lastApprovalFor = ms.getSnapshotKey().getSnapshotStatus().toString();
                    }

                    // add "approve as" status indicator to response record, if not blank
                    if (!StringUtils.isBlank(lastApprovalFor))
                        ((ObjectNode) objNode).put("approved_as", lastApprovalFor);

                    String systemStatus = "";
                    List<MetadataSnapshot> resultsSystem = querySystemSnapshot.setMaxResults(1).getResultList();
                    for (MetadataSnapshot ms : resultsSystem) {
                        systemStatus = ms.getSnapshotKey().getSnapshotStatus().toString();
                    }

                    // add "system status" indicator to response record, if not blank
                    if (!StringUtils.isBlank(lastApprovalFor))
                        ((ObjectNode) objNode).put("system_status", systemStatus);
                }

                recordsObject.put("total", rowCount);
            }

            return Response.status(Response.Status.OK).entity(recordsObject.toString()).build();
        } finally {
            em.close();
        }
    }

    /**
     * Acquire a List of records in pending ("Submitted") state, to be approved
     * for indexing and searching.
     *
     * JSON response is of the form:
     *
     * {"records":[{"code_id":n, ...} ],
     *  "start":0, "rows":20, "total":100}
     *
     * Where records is an array of DOECodeMetadata JSON, start is the beginning
     * row number, rows is the number requested (or total if less available),
     * and total is the total number of rows matching the filter.
     *
     * Return Codes:
     * 200 - OK, JSON is returned as above
     * 401 - Unauthorized, login is required
     * 403 - Forbidden, insufficient privileges (role required)
     * 500 - unexpected error
     *
     * @param start the starting row number (from 0)
     * @param rows number of rows desired (0 is unlimited)
     * @param siteCode (optional) a SITE OWNERSHIP CODE to filter by site
     * @param state the WORKFLOW STATE if desired (default Submitted and Announced). One of
     * Approved, Saved, Submitted, or Announced, if supplied.
     * @return JSON of a records response
     */
    @GET
    @Path("/projects/pending")
    @Consumes(MediaType.APPLICATION_JSON)
    @Produces(MediaType.APPLICATION_JSON)
    @RequiresAuthentication
    @RequiresRoles("OSTI")
    public Response listProjectsPending(@QueryParam("start") int start, @QueryParam("rows") int rows,
            @QueryParam("site") String siteCode, @QueryParam("state") String state) {
        EntityManager em = DoeServletContextListener.createEntityManager();

        try {
            // get a JPA CriteriaBuilder instance
            CriteriaBuilder cb = em.getCriteriaBuilder();
            // create a CriteriaQuery for the COUNT
            CriteriaQuery<Long> countQuery = cb.createQuery(Long.class);
            Root<DOECodeMetadata> md = countQuery.from(DOECodeMetadata.class);
            countQuery.select(cb.count(md));

            Expression<String> workflowStatus = md.get("workflowStatus");
            Expression<String> siteOwnershipCode = md.get("siteOwnershipCode");

            // default requested STATE; take Submitted and Announced as the default values if not supplied
            List<DOECodeMetadata.Status> requestedStates = new ArrayList();
            String queryState = (StringUtils.isEmpty(state)) ? "" : state.toLowerCase();
            switch (queryState) {
            case "approved":
                requestedStates.add(DOECodeMetadata.Status.Approved);
                break;
            case "saved":
                requestedStates.add(DOECodeMetadata.Status.Saved);
                break;
            case "submitted":
                requestedStates.add(DOECodeMetadata.Status.Submitted);
                break;
            case "announced":
                requestedStates.add(DOECodeMetadata.Status.Announced);
                break;
            default:
                requestedStates.add(DOECodeMetadata.Status.Submitted);
                requestedStates.add(DOECodeMetadata.Status.Announced);
                break;
            }

            Predicate statusPredicate = workflowStatus.in(requestedStates);
            ParameterExpression<String> site = cb.parameter(String.class, "site");

            if (null == siteCode) {
                countQuery.where(statusPredicate);
            } else {
                countQuery.where(cb.and(statusPredicate, cb.equal(siteOwnershipCode, site)));
            }
            // query for the COUNT
            TypedQuery<Long> cq = em.createQuery(countQuery);
            cq.setParameter("status", requestedStates);
            if (null != siteCode)
                cq.setParameter("site", siteCode);

            long rowCount = cq.getSingleResult();
            // rows count should be less than 100 for pagination; 0 is a special case
            rows = (rows > 100) ? 100 : rows;

            // create a CriteriaQuery for the ROWS
            CriteriaQuery<DOECodeMetadata> rowQuery = cb.createQuery(DOECodeMetadata.class);
            rowQuery.select(md);

            if (null == siteCode) {
                rowQuery.where(statusPredicate);
            } else {
                rowQuery.where(cb.and(statusPredicate, cb.equal(siteOwnershipCode, site)));
            }

            TypedQuery<DOECodeMetadata> rq = em.createQuery(rowQuery);
            rq.setParameter("status", requestedStates);
            if (null != siteCode)
                rq.setParameter("site", siteCode);
            rq.setFirstResult(start);
            if (0 != rows)
                rq.setMaxResults(rows);

            RecordsList records = new RecordsList(rq.getResultList());
            records.setTotal(rowCount);
            records.setStart(start);

            return Response.ok().entity(mapper.valueToTree(records).toString()).build();
        } finally {
            em.close();
        }
    }

    /**
     * Call to auto-populate Metadata information via Connector, if possible.
     *
     * @param url the REPOSITORY URL to look up information from
     * @param format optionally, the output format ("yaml" supported) JSON is default
     * @return a Metadata instance in the desired output format if information was found
     */
    @GET
    @Path("/autopopulate")
    @Produces({ MediaType.APPLICATION_JSON, "text/yaml" })
    public Response autopopulate(@QueryParam("repo") String url, @QueryParam("format") String format) {
        JsonNode resultJson = factory.read(url);

        if (null == resultJson)
            return Response.status(Response.Status.NO_CONTENT).build();

        ObjectNode result = (ObjectNode) resultJson;
        result.remove("code_id");

        // if YAML is requested, return that; otherwise, default to JSON output
        if ("yaml".equals(format)) {
            try {
                return Response.status(Response.Status.OK)
                        .header("Content-Disposition", "attachment; filename = \"metadata.yml\"")
                        .header("Content-Type", "text/yaml").entity(HttpUtil.writeMetadataYaml(result)).build();
            } catch (IOException e) {
                log.warn("YAML conversion error: " + e.getMessage());
                return ErrorResponse.status(Response.Status.INTERNAL_SERVER_ERROR, "YAML conversion error.")
                        .build();
            }
        } else {
            // send back the default JSON response
            return Response.ok().header("Content-Type", MediaType.APPLICATION_JSON)
                    .entity(mapper.createObjectNode().putPOJO("metadata", result).toString()).build();
        }
    }

    /**
     * Persist the DOECodeMetadata Object to the persistence layer.  Assumes an
     * open Transaction is already in progress, and it's up to the caller to
     * handle Exceptions or commit as appropriate.
     *
     * If the "code ID" is already present in the Object to store, it will
     * attempt to merge changes; otherwise, a new Object will be instantiated
     * in the database.  Note that any WORKFLOW STATUS present will be preserved,
     * regardless of the incoming one.
     *
     * @param em the EntityManager to interface with the persistence layer
     * @param md the Object to store
     * @param user the User performing this action (must be the OWNER of the
     * record in order to UPDATE)
     * @throws NotFoundException when record to update is not on file
     * @throws IllegalAccessException when attempting to update record not
     * owned by User
     * @throws InvocationTargetException on reflection errors
     */
    private void store(EntityManager em, DOECodeMetadata md, User user)
            throws NotFoundException, IllegalAccessException, InvocationTargetException {
        // fix the open source value before storing
        md.setOpenSource(
                Accessibility.OS.equals(md.getAccessibility()) || Accessibility.ON.equals(md.getAccessibility()));

        ValidatorFactory validators = javax.validation.Validation.buildDefaultValidatorFactory();
        Validator validator = validators.getValidator();

        // must be OSTI user in order to add/update PROJECT KEYWORDS
        List<String> projectKeywords = md.getProjectKeywords();
        if (projectKeywords != null && !projectKeywords.isEmpty() && !user.hasRole("OSTI"))
            throw new ValidationException("Project Keywords can only be set by authorized users.");

        // if there's a CODE ID, attempt to look up the record first and
        // copy attributes into it
        if (null == md.getCodeId() || 0 == md.getCodeId()) {
            // perform length validations on Bean
            Set<ConstraintViolation<DOECodeMetadata>> violations = validator.validate(md);
            if (!violations.isEmpty()) {
                List<String> reasons = new ArrayList<>();

                violations.stream().forEach(violation -> {
                    reasons.add(violation.getMessage());
                });
                throw new BadRequestException(ErrorResponse.badRequest(reasons).build());
            }
            em.persist(md);
        } else {
            DOECodeMetadata emd = em.find(DOECodeMetadata.class, md.getCodeId());

            if (null != emd) {
                // must be the OWNER, SITE ADMIN, or OSTI in order to UPDATE
                if (!user.getEmail().equals(emd.getOwner()) && !user.hasRole(emd.getSiteOwnershipCode())
                        && !user.hasRole("OSTI"))
                    throw new IllegalAccessException("Invalid access attempt.");

                // to Save, item must be non-existant, or already in Saved workflow status (if here, we know it exists)
                if (Status.Saved.equals(md.getWorkflowStatus()) && !Status.Saved.equals(emd.getWorkflowStatus()))
                    throw new BadRequestException(ErrorResponse
                            .badRequest("Save cannot be performed after a record has been Submitted or Announced.")
                            .build());

                // these fields WILL NOT CHANGE on edit/update
                md.setOwner(emd.getOwner());
                md.setSiteOwnershipCode(emd.getSiteOwnershipCode());
                // if there's ALREADY a DOI, and we have been SUBMITTED/APPROVED, keep it
                if (StringUtils.isNotEmpty(emd.getDoi()) && (Status.Submitted.equals(emd.getWorkflowStatus())
                        || Status.Approved.equals(emd.getWorkflowStatus())))
                    md.setDoi(emd.getDoi());

                // do not modify AutoBackfill RI info
                List<RelatedIdentifier> originalList = emd.getRelatedIdentifiers();
                List<RelatedIdentifier> newList = md.getRelatedIdentifiers();
                // if there is a New List and a non-empty Original List, then process RI info
                if (newList != null && originalList != null && !originalList.isEmpty()) {
                    // get AutoBackfill data
                    List<RelatedIdentifier> autoRIList = getSourceRi(originalList,
                            RelatedIdentifier.Source.AutoBackfill);

                    // restore any modified Auto data
                    newList.removeAll(autoRIList); // always remove match
                    newList.addAll(autoRIList); // add back, if needed

                    md.setRelatedIdentifiers(newList);
                }

                // perform length validations on Bean
                Set<ConstraintViolation<DOECodeMetadata>> violations = validator.validate(md);
                if (!violations.isEmpty()) {
                    List<String> reasons = new ArrayList<>();

                    violations.stream().forEach(violation -> {
                        reasons.add(violation.getMessage());
                    });
                    throw new BadRequestException(ErrorResponse.badRequest(reasons).build());
                }

                // found it, "merge" Bean attributes
                BeanUtilsBean noNulls = new NoNullsBeanUtilsBean();
                noNulls.copyProperties(emd, md);

                // if the RELEASE DATE was set, it might have been "cleared" (set to null)
                // and thus ignored by the Bean copy; this sets the value regardless if setReleaseDate() got called
                if (md.hasSetReleaseDate())
                    emd.setReleaseDate(md.getReleaseDate());

                // what comes back needs to be complete:
                noNulls.copyProperties(md, emd);

                // EntityManager should handle this attached Object
                // NOTE: the returned Object is NOT ATTACHED to the EntityManager
            } else {
                // can't find record to update, that's an error
                log.warn("Unable to locate record for " + md.getCodeId() + " to update.");
                throw new NotFoundException("Record Code ID " + md.getCodeId() + " not on file.");
            }
        }
    }

    /**
     * Convert incoming JSON object of Metadata information to YAML if possible.
     *
     * @param object JSON of the Metadata information
     * @return YAML of that JSON object, if mappable
     */
    @POST
    @Consumes(MediaType.APPLICATION_JSON)
    @Produces("text/yaml")
    @Path("/yaml")
    public Response asYAML(String object) {
        try {
            DOECodeMetadata md = DOECodeMetadata.parseJson(new StringReader(object));

            return Response.status(Response.Status.OK).entity(HttpUtil.writeMetadataYaml(md)).build();
        } catch (IOException e) {
            log.warn("YAML conversion error: " + e.getMessage());
            return ErrorResponse.internalServerError("YAML conversion error.").build();
        }
    }

    /**
     * Send this Metadata to the ARCHIVER external support process.
     *
     * Needs a CODE ID and one of either an ARCHIVE FILE or REPOSITORY LINK.
     *
     * If nothing supplied to archive, do nothing.
     *
     * @param codeId the CODE ID for this METADATA
     * @param repositoryLink (optional) the REPOSITORY LINK value, or null if none
     * @param archiveFile (optional) the File recently uploaded to ARCHIVE, or null if none
     * @param archiveContainer (optional) the Container recently uploaded to ARCHIVE, or null if none
     * @throws IOException on IO transmission errors
     */
    private static void sendToArchiver(Long codeId, String repositoryLink, File archiveFile, File archiveContainer)
            throws IOException {
        if ("".equals(ARCHIVER_URL))
            return;

        // Nothing sent?
        if (StringUtils.isBlank(repositoryLink) && null == archiveFile && null == archiveContainer)
            return;

        // set up a connection
        CloseableHttpClient hc = HttpClientBuilder.create().setDefaultRequestConfig(RequestConfig.custom()
                .setSocketTimeout(300000).setConnectTimeout(300000).setConnectionRequestTimeout(300000).build())
                .build();

        try {
            HttpPost post = new HttpPost(ARCHIVER_URL);
            // attributes to send
            ObjectNode request = mapper.createObjectNode();
            request.put("code_id", codeId);
            request.put("repository_link", repositoryLink);

            // determine if there's a file to send or not
            if (null == archiveFile && null == archiveContainer) {
                post.setHeader("Content-Type", "application/json");
                post.setHeader("Accept", "application/json");

                post.setEntity(new StringEntity(request.toString(), "UTF-8"));
            } else {
                MultipartEntityBuilder mpe = MultipartEntityBuilder.create()
                        .setMode(HttpMultipartMode.BROWSER_COMPATIBLE)
                        .addPart("project", new StringBody(request.toString(), ContentType.APPLICATION_JSON));

                if (archiveFile != null)
                    mpe.addPart("file", new FileBody(archiveFile, ContentType.DEFAULT_BINARY));

                if (archiveContainer != null)
                    mpe.addPart("container", new FileBody(archiveContainer, ContentType.DEFAULT_BINARY));

                post.setEntity(mpe.build());
            }
            HttpResponse response = hc.execute(post);

            int statusCode = response.getStatusLine().getStatusCode();

            if (HttpStatus.SC_OK != statusCode && HttpStatus.SC_CREATED != statusCode) {
                throw new IOException("Archiver Error: " + EntityUtils.toString(response.getEntity()));
            }
        } catch (IOException e) {
            log.warn("Archiver request error: " + e.getMessage());
            throw e;
        } finally {
            try {
                if (null != hc)
                    hc.close();
            } catch (IOException e) {
                log.warn("Close Error: " + e.getMessage());
            }
        }
    }

    /**
     * Validate accepted file types.
     *
     * @param fileName the uploaded filename to evaluate.
     * @param containerName the uploaded container image filename to evaluate.
     */
    private static void validateUploads(FormDataContentDisposition fileInfo,
            FormDataContentDisposition containerInfo) {
        // evaluate file upload
        if (fileInfo != null && !StringUtils.isBlank(fileInfo.getFileName())) {
            String fileName = fileInfo.getFileName();
            Pattern filePattern = Pattern.compile("[.](?:zip|tgz|tar(?:[.](?:gz|bz2))?)$");
            Matcher m = filePattern.matcher(fileName);
            if (!m.find())
                throw new ValidationException(
                        "File upload failed!  File must be of type: .zip, .tar, .tgz, .tar.gz, .tar.bz2");
        }

        // evaluate container upload
        if (containerInfo != null && !StringUtils.isBlank(containerInfo.getFileName())) {
            String fileName = containerInfo.getFileName();
            Pattern containerPattern = Pattern.compile("[.](?:simg|tgz|tar(?:[.]gz)?)$");
            Matcher m = containerPattern.matcher(fileName);
            if (!m.find())
                throw new ValidationException(
                        "Container image upload failed!  File must be of type: .tar, .tgz, .tar.gz, .simg");
        }
    }

    /**
     * Get specific Source RI from metadata.
     *
     * @param md the Metadata to evaluate.
     * @return Updated DOECodeMetadata object.
     */
    private static List<RelatedIdentifier> getSourceRi(List<RelatedIdentifier> list,
            RelatedIdentifier.Source source) {
        // get detached list of RI to check
        List<RelatedIdentifier> riList = new ArrayList<>();
        riList.addAll(list);

        // filter to targeted RI
        riList = riList.stream().filter(p -> p.getSource() == source).collect(Collectors.toList());

        return riList;
    }

    /**
     * Remove non-indexable New/Previous RI from metadata.
     *
     * @param em the EntityManager to control commits.
     * @param md the Metadata to evaluate.
     * @return Updated DOECodeMetadata object.
     */
    private static DOECodeMetadata removeNonIndexableRi(EntityManager em, DOECodeMetadata md) throws IOException {
        // need a detached copy of the RI data
        DOECodeMetadata alteredMd = new DOECodeMetadata();
        BeanUtilsBean bean = new BeanUtilsBean();

        try {
            bean.copyProperties(alteredMd, md);
        } catch (IllegalAccessException | InvocationTargetException ex) {
            // log issue, swallow error
            String msg = "NonIndexable RI Removal Bean Error: " + ex.getMessage();
            throw new IOException(msg);
        }

        TypedQuery<MetadataSnapshot> querySnapshot = em
                .createNamedQuery("MetadataSnapshot.findByDoiAndStatus", MetadataSnapshot.class)
                .setParameter("status", DOECodeMetadata.Status.Approved);

        // get detached list of RI to check
        List<RelatedIdentifier> riList = new ArrayList<>();
        riList.addAll(alteredMd.getRelatedIdentifiers());

        // filter to targeted RI
        List<RelatedIdentifier> filteredRiList = riList.stream()
                .filter(p -> p.getIdentifierType() == RelatedIdentifier.Type.DOI
                        && (p.getRelationType() == RelatedIdentifier.RelationType.IsNewVersionOf
                                || p.getRelationType() == RelatedIdentifier.RelationType.IsPreviousVersionOf))
                .collect(Collectors.toList());

        // track removals
        List<RelatedIdentifier> removalList = new ArrayList<>();

        for (RelatedIdentifier ri : filteredRiList) {
            // lookup by Snapshot by current DOI
            querySnapshot.setParameter("doi", ri.getIdentifierValue());

            List<MetadataSnapshot> results = querySnapshot.getResultList();

            // if no results, keep, otherwise remove unless there is a minted version found
            boolean remove = !results.isEmpty();
            for (MetadataSnapshot ms : results) {
                if (ms.getDoiIsMinted()) {
                    remove = false;
                    break;
                }
            }

            if (remove)
                removalList.add(ri);
        }

        // perform removals, as needed, and update
        if (!removalList.isEmpty()) {
            riList.removeAll(removalList);
            alteredMd.setRelatedIdentifiers(riList);
        }

        return alteredMd;
    }

    /**
     * Get previous snapshot info for use in backfill process that occurs after current snapshot is updated.
     *
     * @param em the EntityManager to control commits.
     * @param md the Metadata to evaluate for RI backfilling.
     * @return List of RelatedIdentifier objects.
     */
    private List<RelatedIdentifier> getPreviousRiList(EntityManager em, DOECodeMetadata md) throws IOException {
        // if current project has no DOI, there is nothing to process later on, so do not pull previous info
        if (StringUtils.isBlank(md.getDoi()))
            return null;

        long codeId = md.getCodeId();

        // pull last know Approved info
        TypedQuery<MetadataSnapshot> querySnapshot = em
                .createNamedQuery("MetadataSnapshot.findByCodeIdAndStatus", MetadataSnapshot.class)
                .setParameter("codeId", codeId).setParameter("status", DOECodeMetadata.Status.Approved);

        List<MetadataSnapshot> results = querySnapshot.setMaxResults(1).getResultList();

        // get previous Approved RI list, if applicable
        List<RelatedIdentifier> previousList = new ArrayList<>();
        for (MetadataSnapshot ms : results) {
            try {
                DOECodeMetadata pmd = DOECodeMetadata.parseJson(new StringReader(ms.getJson()));

                previousList = pmd.getRelatedIdentifiers();

                if (previousList == null)
                    previousList = new ArrayList<>();

                // filter to targeted RI
                previousList = previousList.stream().filter(p -> p.getIdentifierType() == RelatedIdentifier.Type.DOI
                        && (p.getRelationType() == RelatedIdentifier.RelationType.IsNewVersionOf
                                || p.getRelationType() == RelatedIdentifier.RelationType.IsPreviousVersionOf))
                        .collect(Collectors.toList());

            } catch (IOException ex) {
                // unable to parse JSON, but for this process
                String msg = "Unable to parse previously 'Approved' Snapshot JSON for " + codeId + ": "
                        + ex.getMessage();
                throw new IOException(msg);
            }
            break; // failsafe: there should only ever be one, at most
        }

        return previousList;
    }

    /**
     * Add/Remove backfill RI information.
     * To modify source items, you must be an OSTI admin, project owner, or site admin.
     *
     * @param em the EntityManager to control commits.
     * @param md the Metadata to evaluate for RI updating.
     * @param previousList the RelatedIdentifiers from previous Approval.
     */
    private void backfillProjects(EntityManager em, DOECodeMetadata md, List<RelatedIdentifier> previousList)
            throws IllegalAccessException, IOException {
        // if current project has no DOI, there is nothing to process
        if (StringUtils.isBlank(md.getDoi()))
            return;

        // get current list of RI info, for backfill additions
        List<RelatedIdentifier> additionList = md.getRelatedIdentifiers();

        if (additionList == null)
            additionList = new ArrayList<>();

        // filter additions to targeted RI
        additionList = additionList.stream()
                .filter(p -> p.getIdentifierType() == RelatedIdentifier.Type.DOI
                        && (p.getRelationType() == RelatedIdentifier.RelationType.IsNewVersionOf
                                || p.getRelationType() == RelatedIdentifier.RelationType.IsPreviousVersionOf))
                .collect(Collectors.toList());

        if (previousList == null)
            previousList = new ArrayList<>();

        // previous relations no longer defined must be removed
        previousList.removeAll(additionList);

        // store details about what will need sent to OSTI and re-indexed
        Map<Long, DOECodeMetadata> backfillSendToIndex = new HashMap<>();
        Map<Long, DOECodeMetadata> backfillSendToOsti = new HashMap<>();

        // define needed queries
        TypedQuery<DOECodeMetadata> deleteQuery = em.createNamedQuery("DOECodeMetadata.findByDoiAndRi",
                DOECodeMetadata.class);
        TypedQuery<DOECodeMetadata> addQuery = em.createNamedQuery("DOECodeMetadata.findByDoi",
                DOECodeMetadata.class);
        TypedQuery<MetadataSnapshot> snapshotQuery = em.createNamedQuery("MetadataSnapshot.findByCodeIdAndStatus",
                MetadataSnapshot.class);
        TypedQuery<MetadataSnapshot> querySnapshot = em
                .createNamedQuery("MetadataSnapshot.findByCodeIdLastNotStatus", MetadataSnapshot.class)
                .setParameter("status", DOECodeMetadata.Status.Approved);

        List<RelatedIdentifier> backfillSourceList;

        // for each BackfillType, perform actions:  delete obsolete previous info / add new info
        for (RelatedIdentifier.BackfillType backfillType : RelatedIdentifier.BackfillType.values()) {
            // previous relations no longer defined must be removed, current relations need to be added
            backfillSourceList = backfillType == RelatedIdentifier.BackfillType.Deletion ? previousList
                    : additionList;

            // if there is no list to process, skip
            if (backfillSourceList == null || backfillSourceList.isEmpty())
                continue;

            for (RelatedIdentifier ri : backfillSourceList) {
                // get inverse relation
                RelatedIdentifier inverseRelation = new RelatedIdentifier(ri);
                inverseRelation.setRelationType(ri.getRelationType().inverse());
                inverseRelation.setIdentifierValue(md.getDoi());
                inverseRelation.setSource(RelatedIdentifier.Source.AutoBackfill);

                List<RelatedIdentifier> targetedList = Arrays.asList(inverseRelation);

                List<DOECodeMetadata> results = new ArrayList<>();
                List<MetadataSnapshot> snapshotResults;

                if (backfillType == RelatedIdentifier.BackfillType.Deletion) {
                    // check for the existance of the inverse relation
                    deleteQuery.setParameter("doi", ri.getIdentifierValue())
                            .setParameter("type", inverseRelation.getIdentifierType())
                            .setParameter("value", inverseRelation.getIdentifierValue())
                            .setParameter("relType", inverseRelation.getRelationType());

                    results = deleteQuery.getResultList();
                } else if (backfillType == RelatedIdentifier.BackfillType.Addition) {
                    // lookup target DOI
                    addQuery.setParameter("doi", ri.getIdentifierValue());

                    results = addQuery.getResultList();
                }

                // update RI where needed
                for (DOECodeMetadata bmd : results) {
                    // target CODE ID and Workflow Status
                    Long codeId = bmd.getCodeId();
                    DOECodeMetadata.Status status = bmd.getWorkflowStatus();

                    List<RelatedIdentifier> updateList = bmd.getRelatedIdentifiers();

                    if (updateList == null)
                        updateList = new ArrayList<>();

                    // get User data
                    List<RelatedIdentifier> userRIList = getSourceRi(updateList, RelatedIdentifier.Source.User);

                    // update metadata RI info
                    updateList.removeAll(targetedList); // always remove match
                    if (backfillType == RelatedIdentifier.BackfillType.Addition)
                        updateList.addAll(targetedList); // add back, if needed

                    // restore any modified User data
                    updateList.removeAll(userRIList); // always remove match
                    updateList.addAll(userRIList); // add back, if needed

                    // save changes
                    bmd.setRelatedIdentifiers(updateList);

                    // update snapshot metadata
                    snapshotQuery.setParameter("codeId", codeId).setParameter("status", status);

                    snapshotResults = snapshotQuery.getResultList();

                    // update snapshot RI, for same status, where needed
                    for (MetadataSnapshot ms : snapshotResults) {
                        try {
                            DOECodeMetadata smd = DOECodeMetadata.parseJson(new StringReader(ms.getJson()));

                            List<RelatedIdentifier> snapshotList = smd.getRelatedIdentifiers();

                            if (snapshotList == null)
                                snapshotList = new ArrayList<>();

                            // get User data
                            userRIList = getSourceRi(snapshotList, RelatedIdentifier.Source.User);

                            // update snapshot RI info, if needed
                            snapshotList.removeAll(targetedList); // always remove match
                            if (backfillType == RelatedIdentifier.BackfillType.Addition)
                                snapshotList.addAll(targetedList); // add back, if needed

                            // restore any modified User data
                            snapshotList.removeAll(userRIList); // always remove match
                            snapshotList.addAll(userRIList); // add back, if needed

                            // save changes to Snapshot
                            smd.setRelatedIdentifiers(snapshotList);
                            ms.setJson(smd.toJson().toString());

                            // log updated, Approved snapshot info for post-backfill actions
                            if (status == DOECodeMetadata.Status.Approved) {
                                // log for re-indexing
                                backfillSendToIndex.put(codeId, smd);

                                // lookup snapshot status info, prior to Approval
                                querySnapshot.setParameter("codeId", codeId);

                                List<MetadataSnapshot> previousResults = querySnapshot.setMaxResults(1)
                                        .getResultList();
                                for (MetadataSnapshot pms : previousResults) {
                                    DOECodeMetadata.Status lastApprovalFor = pms.getSnapshotKey()
                                            .getSnapshotStatus();

                                    // if Approved for Announcement, log for OSTI
                                    if (lastApprovalFor == DOECodeMetadata.Status.Announced)
                                        backfillSendToOsti.put(codeId, smd);

                                    break; // failsafe, but should only be at most one item returned
                                }
                            }
                        } catch (IOException ex) {
                            // unable to parse JSON, but for this process
                            String msg = "Unable to parse '" + ms.getSnapshotKey().getSnapshotStatus()
                                    + "' Snapshot JSON for " + ms.getSnapshotKey().getCodeId() + ": "
                                    + ex.getMessage();
                            throw new IOException(msg);
                        }
                    }
                }
            }
        }

        // update OSTI, as needed
        for (Map.Entry<Long, DOECodeMetadata> entry : backfillSendToOsti.entrySet()) {
            sendToOsti(em, entry.getValue());
        }

        // update Index, as needed
        for (Map.Entry<Long, DOECodeMetadata> entry : backfillSendToIndex.entrySet()) {
            sendToIndex(em, entry.getValue());
        }
    }

    /**
     * Attempt to send this Metadata information to the indexing service configured.
     * If no service is configured, do nothing.
     *
     * @param em the related EntityManager
     * @param md the Metadata to send
     */
    private static void sendToIndex(EntityManager em, DOECodeMetadata md) {
        // if indexing is not configured, skip this step
        if ("".equals(INDEX_URL))
            return;

        // set some reasonable default timeouts
        RequestConfig rc = RequestConfig.custom().setSocketTimeout(60000).setConnectTimeout(60000)
                .setConnectionRequestTimeout(60000).build();
        // create an HTTP client to request through
        CloseableHttpClient hc = HttpClientBuilder.create().setDefaultRequestConfig(rc).build();
        try {
            // do not index DOE CODE New/Previous DOI related identifiers if Approved without a Release Date
            DOECodeMetadata indexableMd = removeNonIndexableRi(em, md);

            // construct a POST submission to the indexer service
            HttpPost post = new HttpPost(INDEX_URL);
            post.setHeader("Content-Type", "application/json");
            post.setHeader("Accept", "application/json");
            // add JSON String to index for later display/search
            ObjectNode node = (ObjectNode) index_mapper.valueToTree(indexableMd);
            node.put("json", indexableMd.toJson().toString());
            post.setEntity(new StringEntity(node.toString(), "UTF-8"));

            HttpResponse response = hc.execute(post);

            if (HttpStatus.SC_OK != response.getStatusLine().getStatusCode()) {
                log.warn("Indexing Error occurred for ID=" + md.getCodeId());
                log.warn("Message: " + EntityUtils.toString(response.getEntity()));
            }
        } catch (IOException e) {
            log.warn("Indexing Error: " + e.getMessage() + " ID=" + md.getCodeId());
        } finally {
            try {
                if (null != hc)
                    hc.close();
            } catch (IOException e) {
                log.warn("Index Close Error: " + e.getMessage());
            }
        }
    }

    /**
     * Perform SAVE workflow on indicated METADATA.
     *
     * @param json the JSON String containing the metadata to SAVE
     * @param file a FILE associated with this record if any
     * @param fileInfo the FILE disposition information if any
     * @param container a CONTAINER IMAGE associated with this record if any
     * @param containerInfo the CONTAINER IMAGE disposition of information if any
     * @return a Response containing the JSON of the saved record if successful,
     * or error information if not
     */
    private Response doSave(String json, InputStream file, FormDataContentDisposition fileInfo,
            InputStream container, FormDataContentDisposition containerInfo) {
        EntityManager em = DoeServletContextListener.createEntityManager();
        Subject subject = SecurityUtils.getSubject();
        User user = (User) subject.getPrincipal();

        try {
            validateUploads(fileInfo, containerInfo);

            DOECodeMetadata md = DOECodeMetadata.parseJson(new StringReader(json));

            em.getTransaction().begin();

            performDataNormalization(md);

            md.setWorkflowStatus(Status.Saved); // default to this
            md.setOwner(user.getEmail()); // this User should OWN it
            md.setSiteOwnershipCode(user.getSiteId());

            store(em, md, user);

            // re-attach metadata to transaction in order to store any changes beyond this point
            md = em.find(DOECodeMetadata.class, md.getCodeId());

            // if there's a FILE associated here, store it
            if (null != file && null != fileInfo) {
                try {
                    String fileName = writeFile(file, md.getCodeId(), fileInfo.getFileName(), FILE_UPLOADS);
                    md.setFileName(fileName);
                } catch (IOException e) {
                    log.error("File Upload Failed: " + e.getMessage());
                    return ErrorResponse.internalServerError("File upload failed.").build();
                }
            }

            // if there's a CONTAINER IMAGE associated here, store it
            if (null != container && null != containerInfo) {
                try {
                    String containerName = writeFile(container, md.getCodeId(), containerInfo.getFileName(),
                            CONTAINER_UPLOADS);
                    md.setContainerName(containerName);
                } catch (IOException e) {
                    log.error("Container Image Upload Failed: " + e.getMessage());
                    return ErrorResponse.internalServerError("Container Image upload failed.").build();
                }
            }

            // we're done here
            em.getTransaction().commit();

            return Response.status(200)
                    .entity(mapper.createObjectNode().putPOJO("metadata", md.toJson()).toString()).build();
        } catch (BadRequestException e) {
            return e.getResponse();
        } catch (NotFoundException e) {
            return ErrorResponse.notFound(e.getMessage()).build();
        } catch (IllegalAccessException e) {
            log.warn("Persistence Error:  Invalid update attempt from " + user.getEmail());
            log.warn("Message: " + e.getMessage());
            return ErrorResponse.forbidden("Unable to persist update for indicated record.").build();
        } catch (ValidationException e) {
            log.warn("Validation Error: " + e.getMessage());
            return ErrorResponse.badRequest(e.getMessage()).build();
        } catch (IOException | InvocationTargetException e) {
            if (em.getTransaction().isActive())
                em.getTransaction().rollback();

            log.warn("Persistence Error: " + e.getMessage());
            return ErrorResponse.internalServerError("Save IO Error: " + e.getMessage()).build();
        } finally {
            em.close();
        }
    }

    /**
     * Handle SUBMIT workflow logic.
     *
     * @param json JSON String containing the METADATA object to SUBMIT
     * @param file (optional) a FILE associated with this METADATA
     * @param fileInfo (optional) the FILE disposition information, if any
     * @param container (optional) a CONTAINER IMAGE associated with this METADATA
     * @param containerInfo (optional) the CONTAINER IMAGE disposition information, if any
     * @return an appropriate Response object to the caller
     */
    private Response doSubmit(String json, InputStream file, FormDataContentDisposition fileInfo,
            InputStream container, FormDataContentDisposition containerInfo) {
        EntityManager em = DoeServletContextListener.createEntityManager();
        Subject subject = SecurityUtils.getSubject();
        User user = (User) subject.getPrincipal();

        try {
            validateUploads(fileInfo, containerInfo);

            DOECodeMetadata md = DOECodeMetadata.parseJson(new StringReader(json));

            Long currentCodeId = md.getCodeId();
            boolean previouslySaved = false;
            if (currentCodeId != null) {
                DOECodeMetadata emd = em.find(DOECodeMetadata.class, currentCodeId);

                if (emd != null)
                    previouslySaved = Status.Saved.equals(emd.getWorkflowStatus());
            }

            // lookup Announced Snapshot status
            TypedQuery<MetadataSnapshot> querySnapshot = em
                    .createNamedQuery("MetadataSnapshot.findByCodeIdAndStatus", MetadataSnapshot.class)
                    .setParameter("codeId", currentCodeId).setParameter("status", DOECodeMetadata.Status.Announced);

            List<MetadataSnapshot> results = querySnapshot.setMaxResults(1).getResultList();
            if (results.size() > 0) {
                log.error("Cannot Submit, Previously Announced: " + currentCodeId);
                return ErrorResponse.internalServerError(
                        "This record was previously Announced to E-Link, if you need to update the metadata, please change your endpoint to \"/announce.\"")
                        .build();
            }

            em.getTransaction().begin();

            performDataNormalization(md);

            // set the ownership and workflow status
            md.setOwner(user.getEmail());
            md.setWorkflowStatus(Status.Submitted);
            md.setSiteOwnershipCode(user.getSiteId());

            // store it
            store(em, md, user);

            // re-attach metadata to transaction in order to store any changes beyond this point
            md = em.find(DOECodeMetadata.class, md.getCodeId());

            // if there's a FILE associated here, store it
            String fullFileName = "";
            if (null != file && null != fileInfo) {
                try {
                    fullFileName = writeFile(file, md.getCodeId(), fileInfo.getFileName(), FILE_UPLOADS);
                    md.setFileName(fullFileName);
                } catch (IOException e) {
                    log.error("File Upload Failed: " + e.getMessage());
                    return ErrorResponse.internalServerError("File upload failed.").build();
                }
            }

            // if there's a CONTAINER IMAGE associated here, store it
            String fullContainerName = "";
            if (null != container && null != containerInfo) {
                try {
                    fullContainerName = writeFile(container, md.getCodeId(), containerInfo.getFileName(),
                            CONTAINER_UPLOADS);
                    md.setContainerName(fullContainerName);
                } catch (IOException e) {
                    log.error("Container Image Upload Failed: " + e.getMessage());
                    return ErrorResponse.internalServerError("Container Image upload failed.").build();
                }
            }

            // check validations for Submitted workflow
            List<String> errors = validateSubmit(md);
            if (!errors.isEmpty()) {
                // generate a JSONAPI errors object
                return ErrorResponse.badRequest(errors).build();
            }

            // create OSTI Hosted project, as needed
            try {
                // process local GitLab, if needed
                processOSTIGitLab(md);
            } catch (Exception e) {
                log.error("OSTI GitLab failure: " + e.getMessage());
                return ErrorResponse.internalServerError("Unable to create OSTI Hosted project: " + e.getMessage())
                        .build();
            }

            // send this file upload along to archiver if configured
            try {
                // if no file/container, but previously Saved with a file/container, we need to attach to those streams and send to Archiver
                if (previouslySaved) {
                    if (null == file && !StringUtils.isBlank(md.getFileName())) {
                        java.nio.file.Path destination = Paths.get(FILE_UPLOADS, String.valueOf(md.getCodeId()),
                                md.getFileName());
                        fullFileName = destination.toString();
                        file = Files.newInputStream(destination);
                    }
                    if (null == container && !StringUtils.isBlank(md.getContainerName())) {
                        java.nio.file.Path destination = Paths.get(CONTAINER_UPLOADS,
                                String.valueOf(md.getCodeId()), md.getContainerName());
                        fullContainerName = destination.toString();
                        container = Files.newInputStream(destination);
                    }
                }

                // if a FILE or CONTAINER was sent, create a File Object from it
                File archiveFile = (null == file) ? null : new File(fullFileName);
                File archiveContainer = null; //(null==container) ? null : new File(fullContainerName);
                if (DOECodeMetadata.Accessibility.CO.equals(md.getAccessibility()))
                    // if CO project type, no need to archive the repo because it is local GitLab
                    sendToArchiver(md.getCodeId(), null, archiveFile, archiveContainer);
                else
                    sendToArchiver(md.getCodeId(), md.getRepositoryLink(), archiveFile, archiveContainer);
            } catch (IOException e) {
                log.error("Archiver call failure: " + e.getMessage());
                return ErrorResponse.internalServerError("Unable to archive project.").build();
            }

            // send to DataCite if needed (and there is a RELEASE DATE set)
            if (null != md.getDoi() && null != md.getReleaseDate()) {
                try {
                    DataCite.register(md);
                } catch (IOException e) {
                    // tell why the DataCite registration failed
                    log.warn("DataCite ERROR: " + e.getMessage());
                    return ErrorResponse.internalServerError(
                            "The DOI registration service is currently unavailable, please try to submit your record later. If the issue persists, please contact doecode@osti.gov.")
                            .build();
                }
            }

            // store the snapshot copy of Metadata
            MetadataSnapshot snapshot = new MetadataSnapshot();
            snapshot.getSnapshotKey().setCodeId(md.getCodeId());
            snapshot.getSnapshotKey().setSnapshotStatus(md.getWorkflowStatus());
            snapshot.setDoi(md.getDoi());
            snapshot.setDoiIsMinted(md.getReleaseDate() != null);
            snapshot.setJson(md.toJson().toString());

            em.merge(snapshot);

            // commit it
            em.getTransaction().commit();

            // send NOTIFICATION if configured to do so
            sendStatusNotification(md);

            // we are done here
            return Response.ok().entity(mapper.createObjectNode().putPOJO("metadata", md.toJson()).toString())
                    .build();
        } catch (BadRequestException e) {
            return e.getResponse();
        } catch (NotFoundException e) {
            return ErrorResponse.notFound(e.getMessage()).build();
        } catch (IllegalAccessException e) {
            log.warn("Persistence Error: Unable to update record, invalid owner: " + user.getEmail());
            log.warn("Message: " + e.getMessage());
            return ErrorResponse.forbidden("Logged in User is not allowed to modify this record.").build();
        } catch (ValidationException e) {
            log.warn("Validation Error: " + e.getMessage());
            return ErrorResponse.badRequest(e.getMessage()).build();
        } catch (IOException | InvocationTargetException e) {
            if (em.getTransaction().isActive())
                em.getTransaction().rollback();

            log.warn("Persistence Error Submitting: " + e.getMessage());
            return ErrorResponse.internalServerError("Persistence error submitting record.").build();
        } finally {
            em.close();
        }
    }

    /**
     * Perform ANNOUNCE workflow operation, optionally with associated file uploads.
     *
     * @param json String containing JSON of the Metadata to ANNOUNCE
     * @param file the FILE (if any) to attach to this metadata
     * @param fileInfo file disposition information if FILE present
     * @return a Response containing the JSON of the submitted record if successful, or
     * error information if not
     */
    private Response doAnnounce(String json, InputStream file, FormDataContentDisposition fileInfo,
            InputStream container, FormDataContentDisposition containerInfo) {
        EntityManager em = DoeServletContextListener.createEntityManager();
        Subject subject = SecurityUtils.getSubject();
        User user = (User) subject.getPrincipal();

        try {
            validateUploads(fileInfo, containerInfo);

            DOECodeMetadata md = DOECodeMetadata.parseJson(new StringReader(json));

            Long currentCodeId = md.getCodeId();
            boolean previouslySaved = false;
            if (currentCodeId != null) {
                DOECodeMetadata emd = em.find(DOECodeMetadata.class, currentCodeId);

                if (emd != null)
                    previouslySaved = Status.Saved.equals(emd.getWorkflowStatus());
            }

            em.getTransaction().begin();

            performDataNormalization(md);

            // set the OWNER
            md.setOwner(user.getEmail());
            // set the WORKFLOW STATUS
            md.setWorkflowStatus(Status.Announced);
            // set the SITE
            md.setSiteOwnershipCode(user.getSiteId());
            // if there is NO DOI set, get one
            if (StringUtils.isEmpty(md.getDoi())) {
                DoiReservation reservation = getReservedDoi();
                if (null == reservation)
                    throw new IOException("DOI reservation failure.");
                // set it
                md.setDoi(reservation.getReservedDoi());
            }

            // persist this to the database
            store(em, md, user);

            // re-attach metadata to transaction in order to store any changes beyond this point
            md = em.find(DOECodeMetadata.class, md.getCodeId());

            // if there's a FILE associated here, store it
            String fullFileName = "";
            if (null != file && null != fileInfo) {
                try {
                    fullFileName = writeFile(file, md.getCodeId(), fileInfo.getFileName(), FILE_UPLOADS);
                    md.setFileName(fullFileName);
                } catch (IOException e) {
                    log.error("File Upload Failed: " + e.getMessage());
                    return ErrorResponse.internalServerError("File upload failed.").build();
                }
            }

            // if there's a CONTAINER IMAGE associated here, store it
            String fullContainerName = "";
            if (null != container && null != containerInfo) {
                try {
                    fullContainerName = writeFile(container, md.getCodeId(), containerInfo.getFileName(),
                            CONTAINER_UPLOADS);
                    md.setContainerName(fullContainerName);
                } catch (IOException e) {
                    log.error("Container Image Upload Failed: " + e.getMessage());
                    return ErrorResponse.internalServerError("Container Image upload failed.").build();
                }
            }

            // check validations
            List<String> errors = validateAnnounce(md);
            if (!errors.isEmpty()) {
                return ErrorResponse.badRequest(errors).build();
            }

            // create OSTI Hosted project, as needed
            try {
                // process local GitLab, if needed
                processOSTIGitLab(md);
            } catch (Exception e) {
                log.error("OSTI GitLab failure: " + e.getMessage());
                return ErrorResponse.internalServerError("Unable to create OSTI Hosted project: " + e.getMessage())
                        .build();
            }

            // send this file upload along to archiver if configured
            try {
                // if no file/container, but previously Saved with a file/container, we need to attach to those streams and send to Archiver
                if (previouslySaved) {
                    if (null == file && !StringUtils.isBlank(md.getFileName())) {
                        java.nio.file.Path destination = Paths.get(FILE_UPLOADS, String.valueOf(md.getCodeId()),
                                md.getFileName());
                        fullFileName = destination.toString();
                        file = Files.newInputStream(destination);
                    }
                    if (null == container && !StringUtils.isBlank(md.getContainerName())) {
                        java.nio.file.Path destination = Paths.get(CONTAINER_UPLOADS,
                                String.valueOf(md.getCodeId()), md.getContainerName());
                        fullContainerName = destination.toString();
                        container = Files.newInputStream(destination);
                    }
                }

                // if a FILE or CONTAINER was sent, create a File Object from it
                File archiveFile = (null == file) ? null : new File(fullFileName);
                File archiveContainer = null; //(null==container) ? null : new File(fullContainerName);
                if (DOECodeMetadata.Accessibility.CO.equals(md.getAccessibility()))
                    // if CO project type, no need to archive the repo because it is local GitLab
                    sendToArchiver(md.getCodeId(), null, archiveFile, archiveContainer);
                else
                    sendToArchiver(md.getCodeId(), md.getRepositoryLink(), archiveFile, archiveContainer);
            } catch (IOException e) {
                log.error("Archiver call failure: " + e.getMessage());
                return ErrorResponse.internalServerError("Unable to archive project.").build();
            }

            // send any updates to DataCite as well (if RELEASE DATE is set)
            if (StringUtils.isNotEmpty(md.getDoi()) && null != md.getReleaseDate()) {
                try {
                    DataCite.register(md);
                } catch (IOException e) {
                    // if DataCite registration failed, say why
                    log.warn("DataCite ERROR: " + e.getMessage());
                    return ErrorResponse.internalServerError(
                            "The DOI registration service is currently unavailable, please try to submit your record later. If the issue persists, please contact doecode@osti.gov.")
                            .build();
                }
            }
            // store the snapshot copy of Metadata in SPECIAL STATUS
            MetadataSnapshot snapshot = new MetadataSnapshot();
            snapshot.getSnapshotKey().setCodeId(md.getCodeId());
            snapshot.getSnapshotKey().setSnapshotStatus(md.getWorkflowStatus());
            snapshot.setDoi(md.getDoi());
            snapshot.setDoiIsMinted(md.getReleaseDate() != null);
            snapshot.setJson(md.toJson().toString());

            em.merge(snapshot);

            // if we make it this far, go ahead and commit the transaction
            em.getTransaction().commit();

            // send NOTIFICATION if configured
            sendStatusNotification(md);

            // and we're happy
            return Response.ok().entity(mapper.createObjectNode().putPOJO("metadata", md.toJson()).toString())
                    .build();
        } catch (BadRequestException e) {
            return e.getResponse();
        } catch (NotFoundException e) {
            return ErrorResponse.notFound(e.getMessage()).build();
        } catch (IllegalAccessException e) {
            log.warn("Persistence Error: Invalid owner update attempt: " + user.getEmail());
            log.warn("Message: " + e.getMessage());
            return ErrorResponse.forbidden("Invalid Access: Unable to edit indicated record.").build();
        } catch (ValidationException e) {
            log.warn("Validation Error: " + e.getMessage());
            return ErrorResponse.badRequest(e.getMessage()).build();
        } catch (IOException | InvocationTargetException e) {
            if (em.getTransaction().isActive())
                em.getTransaction().rollback();

            log.warn("Persistence Error: " + e.getMessage());
            return ErrorResponse.internalServerError("IO Error announcing record.").build();
        } finally {
            em.close();
        }
    }

    /**
     * Support multipart-file upload POSTs to SUBMIT.
     *
     * Response Codes:
     * 200 - OK, JSON returned of the metadata information
     * 400 - validation error, errors returned in JSON
     * 401 - authentication is required to POST
     * 403 - access is forbidden to this record
     * 500 - file upload or database operation failed
     *
     * @param metadata contains the JSON of the record metadata information
     * @param file the uploaded file to attach
     * @param fileInfo disposition information for the file name
     * @param container the uploaded container image to attach
     * @param containerInfo disposition information for the container image name
     * @return a Response appropriate to the request status
     */
    @POST
    @Consumes(MediaType.MULTIPART_FORM_DATA)
    @Produces(MediaType.APPLICATION_JSON)
    @Path("/submit")
    @RequiresAuthentication
    public Response submitFile(@FormDataParam("metadata") String metadata, @FormDataParam("file") InputStream file,
            @FormDataParam("file") FormDataContentDisposition fileInfo,
            @FormDataParam("container") InputStream container,
            @FormDataParam("container") FormDataContentDisposition containerInfo) {
        return doSubmit(metadata, file, fileInfo, container, containerInfo);
    }

    /**
     * SUBMIT a record to DOE CODE.
     *
     * Will return a FORBIDDEN attempt should a User attempt to modify someone
     * else's record.
     *
     * @param object JSON of the DOECodeMetadata object to SUBMIT
     * @return a Response containing the persisted metadata entity in JSON
     * @throws InternalServerErrorException on JSON parsing or other IO errors
     */
    @POST
    @Consumes(MediaType.APPLICATION_JSON)
    @Produces(MediaType.APPLICATION_JSON)
    @Path("/submit")
    @RequiresAuthentication
    public Response submit(String object) {
        return doSubmit(object, null, null, null, null);
    }

    /**
     * ANNOUNCE endpoint; saves Software record to DOE CODE and sends results to
     * OSTI ELINK and enters the OSTI workflow.
     *
     * Will return a FORBIDDEN response if the OWNER logged in does not match
     * the record's OWNER.
     *
     * @param object the JSON of the record to ANNOUNCE.
     * @return a Response containing the resulting JSON metadata sent to OSTI,
     * including any DOI registered.
     * @throws InternalServerErrorException on JSON parsing or other IO errors
     */
    @POST
    @Consumes(MediaType.APPLICATION_JSON)
    @Produces(MediaType.APPLICATION_JSON)
    @Path("/announce")
    @RequiresAuthentication
    public Response announce(String object) {
        return doAnnounce(object, null, null, null, null);
    }

    /**
     * Perform ANNOUNCE workflow with associated file upload.
     *
     * Response Codes:
     * 200 - OK, response includes metadata JSON
     * 400 - record validation failed, errors in JSON
     * 401 - Authentication is required to POST
     * 403 - Access is forbidden to this record
     * 500 - JSON parsing error or other unhandled exception
     *
     * @param metadata the METADATA to ANNOUNCE (send to OSTI)
     * @param file a FILE to associate with this METADATA
     * @param fileInfo file disposition information for the FILE
     * @param container a CONTAINER IMAGE to associate with this METADATA
     * @param containerInfo file disposition information for the CONTAINER IMAGE
     * @return a Response containing the metadata, or error information
     */
    @POST
    @Consumes(MediaType.MULTIPART_FORM_DATA)
    @Produces(MediaType.APPLICATION_JSON)
    @Path("/announce")
    @RequiresAuthentication
    public Response announceFile(@FormDataParam("metadata") String metadata,
            @FormDataParam("file") InputStream file, @FormDataParam("file") FormDataContentDisposition fileInfo,
            @FormDataParam("container") InputStream container,
            @FormDataParam("container") FormDataContentDisposition containerInfo) {
        return doAnnounce(metadata, file, fileInfo, container, containerInfo);
    }

    /**
     * POST a Metadata JSON object to the persistence layer.
     * Saves the object to persistence layer; if the entity is already Submitted,
     * this operation is invalid.
     *
     * @param object the JSON to post
     * @return the JSON after persistence; perhaps containing assigned codeId, etc.
     */
    @POST
    @Consumes(MediaType.APPLICATION_JSON)
    @Produces(MediaType.APPLICATION_JSON)
    @RequiresAuthentication
    @Path("/save")
    public Response save(String object) {
        return doSave(object, null, null, null, null);
    }

    /**
     * POST a Metadata to be SAVED with a file upload.
     *
     * @param metadata the JSON containing the Metadata information
     * @param file a FILE associated with this record
     * @param fileInfo file disposition information for the FILE
     * @param container a CONTAINER IMAGE associated with this record
     * @param containerInfo file disposition information for the CONTAINER IMAGE
     * @return a Response containing the JSON of the metadata if successful, or
     * error information if not
     */
    @POST
    @Consumes(MediaType.MULTIPART_FORM_DATA)
    @Produces(MediaType.APPLICATION_JSON)
    @RequiresAuthentication
    @Path("/save")
    public Response save(@FormDataParam("metadata") String metadata, @FormDataParam("file") InputStream file,
            @FormDataParam("file") FormDataContentDisposition fileInfo,
            @FormDataParam("container") InputStream container,
            @FormDataParam("container") FormDataContentDisposition containerInfo) {
        return doSave(metadata, file, fileInfo, container, containerInfo);
    }

    @GET
    @Produces(MediaType.APPLICATION_JSON)
    @Path("/reindex")
    @RequiresAuthentication
    @RequiresRoles("OSTI")
    public Response reindex() throws IOException {
        EntityManager em = DoeServletContextListener.createEntityManager();

        try {
            TypedQuery<MetadataSnapshot> query = em
                    .createNamedQuery("MetadataSnapshot.findAllByStatus", MetadataSnapshot.class)
                    .setParameter("status", DOECodeMetadata.Status.Approved);
            List<MetadataSnapshot> results = query.getResultList();
            int records = 0;

            for (MetadataSnapshot amd : results) {
                DOECodeMetadata md = DOECodeMetadata.parseJson(new StringReader(amd.getJson()));

                sendToIndex(em, md);
                ++records;
            }

            return Response.ok()
                    .entity(mapper.createObjectNode().put("indexed", String.valueOf(records)).toString()).build();
        } finally {
            em.close();
        }
    }

    /**
     * APPROVE endpoint; sends the Metadata of a targeted project to Index.
     *
     * Will return a FORBIDDEN response if the OWNER logged in does not match
     * the record's OWNER.
     *
     * @param codeId the CODE ID of the record to APPROVE.
     * @return a Response containing the JSON of the approved record if successful, or
     * error information if not
     * @throws InternalServerErrorException on JSON parsing or other IO errors
     */
    @GET
    @Path("/approve/{codeId}")
    @Produces(MediaType.APPLICATION_JSON)
    @RequiresAuthentication
    @RequiresRoles("OSTI")
    public Response approve(@PathParam("codeId") Long codeId) {
        EntityManager em = DoeServletContextListener.createEntityManager();
        Subject subject = SecurityUtils.getSubject();
        User user = (User) subject.getPrincipal();

        try {
            DOECodeMetadata md = em.find(DOECodeMetadata.class, codeId);

            if (null == md)
                return ErrorResponse.notFound("Code ID not on file.").build();

            // make sure this is Submitted or Announced
            if (!DOECodeMetadata.Status.Submitted.equals(md.getWorkflowStatus())
                    && !DOECodeMetadata.Status.Announced.equals(md.getWorkflowStatus()))
                return ErrorResponse.badRequest("Metadata is not in the Submitted/Announced workflow state.")
                        .build();

            // move Approved Container to downloadable path
            try {
                approveContainerUpload(md);
            } catch (IOException e) {
                log.error("Container move failure: " + e.getMessage());
                return ErrorResponse.internalServerError(e.getMessage()).build();
            }

            // if approving announced, send this to OSTI
            if (DOECodeMetadata.Status.Announced.equals(md.getWorkflowStatus())) {
                sendToOsti(em, md);
            }

            em.getTransaction().begin();
            // set the WORKFLOW STATUS
            md.setWorkflowStatus(Status.Approved);

            // persist this to the database, as validations should already be complete at this stage.
            store(em, md, user);

            // prior to updating snapshot, gather RI List for backfilling
            List<RelatedIdentifier> previousRiList = getPreviousRiList(em, md);

            // store the snapshot copy of Metadata
            MetadataSnapshot snapshot = new MetadataSnapshot();
            snapshot.getSnapshotKey().setCodeId(md.getCodeId());
            snapshot.getSnapshotKey().setSnapshotStatus(md.getWorkflowStatus());
            snapshot.setDoi(md.getDoi());
            snapshot.setDoiIsMinted(md.getReleaseDate() != null);
            snapshot.setJson(md.toJson().toString());

            em.merge(snapshot);

            // perform RI backfilling
            backfillProjects(em, md, previousRiList);

            // if we make it this far, go ahead and commit the transaction
            em.getTransaction().commit();

            // send it to the indexer
            sendToIndex(em, md);

            // send APPROVAL NOTIFICATION to OWNER
            sendApprovalNotification(md);
            sendPOCNotification(md);

            // and we're happy
            return Response.status(Response.Status.OK)
                    .entity(mapper.createObjectNode().putPOJO("metadata", md.toJson()).toString()).build();
        } catch (BadRequestException e) {
            return e.getResponse();
        } catch (NotFoundException e) {
            return ErrorResponse.status(Response.Status.NOT_FOUND, e.getMessage()).build();
        } catch (IllegalAccessException e) {
            log.warn("Persistence Error: Invalid owner update attempt: " + user.getEmail());
            log.warn("Message: " + e.getMessage());
            return ErrorResponse
                    .status(Response.Status.FORBIDDEN, "Invalid Access:  Unable to edit indicated record.").build();
        } catch (IOException | InvocationTargetException e) {
            if (em.getTransaction().isActive())
                em.getTransaction().rollback();

            log.warn("Persistence Error: " + e.getMessage());
            return ErrorResponse
                    .status(Response.Status.INTERNAL_SERVER_ERROR, "IO Error approving record: " + e.getMessage())
                    .build();
        } finally {
            em.close();
        }
    }

    /**
     * Send metadata JSON to OSTI.
     *
     * @param em the related EntityManager
     * @param md the Metadata to send to OSTI
     */
    private void sendToOsti(EntityManager em, DOECodeMetadata md) throws IOException {
        // if configured, post this to OSTI
        String publishing_host = context.getInitParameter("publishing.host");
        if (null != publishing_host) {
            // do not index DOE CODE New/Previous DOI related identifiers if Approved without a Release Date
            DOECodeMetadata indexableMd = removeNonIndexableRi(em, md);

            // set some reasonable default timeouts
            // create an HTTP client to request through
            try (CloseableHttpClient hc = HttpClientBuilder.create().setDefaultRequestConfig(RequestConfig.custom()
                    .setSocketTimeout(60000).setConnectTimeout(60000).setConnectionRequestTimeout(60000).build())
                    .build()) {
                HttpPost post = new HttpPost(publishing_host + "/services/softwarecenter?action=api");
                post.setHeader("Content-Type", "application/json");
                post.setHeader("Accept", "application/json");
                post.setEntity(new StringEntity(mapper.writeValueAsString(indexableMd), "UTF-8"));

                HttpResponse response = hc.execute(post);
                String text = EntityUtils.toString(response.getEntity());

                if (HttpStatus.SC_OK != response.getStatusLine().getStatusCode()) {
                    log.warn("OSTI Error: " + text);
                    throw new IOException("OSTI software publication error for " + md.getCodeId());
                }
            }
        }
    }

    /**
     * Remove duplicate RI entries and normalize values.
     *
     * @param md the Metadata to evaluate
     */
    private void normalizeRelatedIdentifiers(DOECodeMetadata md) {
        List<RelatedIdentifier> currentList = md.getRelatedIdentifiers();

        // nothing to process
        if (currentList == null || currentList.isEmpty())
            return;

        // trim DOI and URL values
        for (RelatedIdentifier ri : currentList)
            if (RelatedIdentifier.Type.DOI.equals(ri.getIdentifierType()))
                ri.setIdentifierValue(trimDoi(ri.getIdentifierValue()));
            else if (RelatedIdentifier.Type.URL.equals(ri.getIdentifierType()))
                ri.setIdentifierValue(trimUrl(ri.getIdentifierValue()));

        // remove RI duplicates
        Set<RelatedIdentifier> s = new HashSet<>();
        s.addAll(currentList);
        currentList.clear();
        currentList.addAll(s);
        md.setRelatedIdentifiers(currentList);
    }

    /**
     * Trim away unneeded DOI prefixes, etc.
     *
     * @param doi the DOI to trim
     */
    private String trimDoi(String doi) {
        // trim DOI down to 10.* variation
        if (!StringUtils.isBlank(doi)) {
            doi = doi.trim();
            Matcher m = DOI_TRIM_PATTERN.matcher(doi);
            if (m.find())
                doi = m.group(1);
        }
        return doi;
    }

    /**
     * Trim away unneeded URL characters, etc.
     *
     * @param url the URL to trim
     */
    private String trimUrl(String url) {
        // remove extra spaces and single trailing slash, if exist
        if (!StringUtils.isBlank(url)) {
            url = url.trim();
            Matcher m = URL_TRIM_PATTERN.matcher(url);
            if (m.find())
                url = m.group(1);
        }
        return url;
    }

    /**
     * Normalize metadata information.
     *
     * @param md the Metadata to evaluate
     */
    private void normalizeMetadata(DOECodeMetadata md) {
        // trim main DOI
        md.setDoi(trimDoi(md.getDoi()));

        // trim main URLs
        md.setRepositoryLink(trimUrl(md.getRepositoryLink()));
        md.setLandingPage(trimUrl(md.getLandingPage()));
        md.setProprietaryUrl(trimUrl(md.getProprietaryUrl()));
        md.setDocumentationUrl(trimUrl(md.getDocumentationUrl()));
    }

    /**
     * Normalize certain aspects of the data prior to storage.
     *
     * @param md the Metadata to normalize
     */
    private void performDataNormalization(DOECodeMetadata md) {
        normalizeMetadata(md);
        normalizeRelatedIdentifiers(md);
    }

    /**
     * Store a File to a specific directory location. All files associated with
     * a CODEID are stored in the same folder.
     *
     * @param in the InputStream containing the file content
     * @param codeId the CODE ID associated with this file content
     * @param fileName the base file name of the file
     * @param basePath the base path destination for the file content
     * @return the absolute filesystem path to the file
     * @throws IOException on IO errors
     */
    private static String writeFile(InputStream in, Long codeId, String fileName, String basePath)
            throws IOException {
        // store this file in a designated base path
        java.nio.file.Path destination = Paths.get(basePath, String.valueOf(codeId), fileName);
        // make intervening folders if needed
        Files.createDirectories(destination.getParent());
        // save it (CLOBBER existing, if one there)
        Files.copy(in, destination, StandardCopyOption.REPLACE_EXISTING);

        return destination.toString();
    }

    /**
     * Convert a File InputStream to a Base64 string.
     *
     * @param in the InputStream containing the file content
     * @return the Base64 string of the file
     * @throws IOException on IO errors
     */
    private static String convertBase64(InputStream in) throws IOException {
        Base64InputStream b64in = new Base64InputStream(in, true);
        return IOUtils.toString(b64in, "UTF-8");
    }

    /**
     * Perform validations for SUBMITTED records.
     *
     * @param m the Metadata information to validate
     * @return a List of error messages if any validation errors, empty if none
     */
    protected static List<String> validateSubmit(DOECodeMetadata m) {
        List<String> reasons = new ArrayList<>();
        if (null == m.getAccessibility())
            reasons.add("Missing Source Accessibility.");
        if (StringUtils.isBlank(m.getSoftwareTitle()))
            reasons.add("Software title is required.");
        if (StringUtils.isBlank(m.getDescription()))
            reasons.add("Description is required.");
        if (null == m.getLicenses())
            reasons.add("A License is required.");
        else if (m.getLicenses().contains(DOECodeMetadata.License.Other.value())
                && StringUtils.isBlank(m.getProprietaryUrl()))
            reasons.add("Proprietary License URL is required.");
        if (null == m.getDevelopers() || m.getDevelopers().isEmpty())
            reasons.add("At least one developer is required.");
        else {
            for (Developer developer : m.getDevelopers()) {
                if (StringUtils.isBlank(developer.getFirstName()))
                    reasons.add("Developer missing first name.");
                if (StringUtils.isBlank(developer.getLastName()))
                    reasons.add("Developer missing last name.");
                if (StringUtils.isNotBlank(developer.getEmail())) {
                    if (!Validation.isValidEmail(developer.getEmail()))
                        reasons.add("Developer email \"" + developer.getEmail() + "\" is not valid.");
                }
                if (StringUtils.isNotBlank(developer.getOrcid())) {
                    if (!Validation.isValidORCID(developer.getOrcid()))
                        reasons.add("Developer ORCID \"" + developer.getOrcid() + "\" is not valid.");
                }
            }
        }
        if (!(null == m.getContributors() || m.getContributors().isEmpty())) {
            for (Contributor contributor : m.getContributors()) {
                if (StringUtils.isNotBlank(contributor.getOrcid())) {
                    if (!Validation.isValidORCID(contributor.getOrcid()))
                        reasons.add("Contributor ORCID \"" + contributor.getOrcid() + "\" is not valid.");
                }
            }
        }
        // if "OS" accessibility, a REPOSITORY LINK is REQUIRED
        if (DOECodeMetadata.Accessibility.OS.equals(m.getAccessibility())) {
            if (StringUtils.isBlank(m.getRepositoryLink()))
                reasons.add("Repository URL is required for open source submissions.");
        } else if (!DOECodeMetadata.Accessibility.CO.equals(m.getAccessibility())) {
            // non-OS & non-CO submissions require a LANDING PAGE (prefix with http:// if missing)
            if (!Validation.isValidUrl(m.getLandingPage()))
                reasons.add("A valid Landing Page URL is required for non-open source submissions.");
        }
        // if repository link is present, and not CO, it needs to be valid too
        if (StringUtils.isNotBlank(m.getRepositoryLink())
                && !DOECodeMetadata.Accessibility.CO.equals(m.getAccessibility())
                && !Validation.isValidRepositoryLink(m.getRepositoryLink()))
            reasons.add("Repository URL is not a valid repository.");
        return reasons;
    }

    /**
     * Perform ANNOUNCE validations on metadata.
     *
     * @param m the Metadata to check
     * @return a List of submission validation errors, empty if none
     */
    protected static List<String> validateAnnounce(DOECodeMetadata m) {
        List<String> reasons = new ArrayList<>();
        // get all the SUBMITTED reasons, if any
        reasons.addAll(validateSubmit(m));
        // add SUBMIT-specific validations
        if (null == m.getReleaseDate())
            reasons.add("Release date is required.");
        if (null == m.getSponsoringOrganizations() || m.getSponsoringOrganizations().isEmpty())
            reasons.add("At least one sponsoring organization is required.");
        else {
            for (SponsoringOrganization o : m.getSponsoringOrganizations()) {
                if (StringUtils.isBlank(o.getOrganizationName()))
                    reasons.add("Sponsoring organization name is required.");
                if (StringUtils.isBlank(o.getPrimaryAward()) && o.isDOE())
                    reasons.add("Primary award number is required.");
                else if (o.isDOE() && !Validation.isValidAwardNumber(o.getPrimaryAward()))
                    reasons.add("Award Number " + o.getPrimaryAward() + " is not valid.");
            }
        }
        if (null == m.getResearchOrganizations() || m.getResearchOrganizations().isEmpty())
            reasons.add("At least one research organization is required.");
        else {
            for (ResearchOrganization o : m.getResearchOrganizations()) {
                if (StringUtils.isBlank(o.getOrganizationName()))
                    reasons.add("Research organization name is required.");
            }
        }
        if (StringUtils.isBlank(m.getRecipientName()))
            reasons.add("Contact name is required.");
        if (StringUtils.isBlank(m.getRecipientEmail()))
            reasons.add("Contact email is required.");
        else {
            if (!Validation.isValidEmail(m.getRecipientEmail()))
                reasons.add("Contact email is not valid.");
        }
        if (StringUtils.isBlank(m.getRecipientPhone()))
            reasons.add("Contact phone number is required.");
        else {
            if (!Validation.isValidPhoneNumber(m.getRecipientPhone()))
                reasons.add("Contact phone number is not valid.");
        }
        if (StringUtils.isBlank(m.getRecipientOrg()))
            reasons.add("Contact organization is required.");

        if (!DOECodeMetadata.Accessibility.OS.equals(m.getAccessibility()))
            if (StringUtils.isBlank(m.getFileName()))
                reasons.add("A file archive must be included for non-open source submissions.");

        return reasons;
    }

    /**
     * Send a NOTIFICATION EMAIL (if configured) when a record is SUBMITTED or
     * ANNOUNCED.
     *
     * @param md the METADATA in question
     */
    private static void sendStatusNotification(DOECodeMetadata md) {
        HtmlEmail email = new HtmlEmail();
        email.setCharset(org.apache.commons.mail.EmailConstants.UTF_8);
        email.setHostName(EMAIL_HOST);

        // if EMAIL or DESTINATION ADDRESS are not set, abort
        if (StringUtils.isEmpty(EMAIL_HOST) || StringUtils.isEmpty(EMAIL_SUBMISSION))
            return;

        // only applicable to SUBMITTED or ANNOUNCED records
        if (!Status.Announced.equals(md.getWorkflowStatus()) && !Status.Submitted.equals(md.getWorkflowStatus()))
            return;

        // get the SITE information
        String siteCode = md.getSiteOwnershipCode();
        Site site = SiteServices.findSiteBySiteCode(siteCode);
        if (null == site) {
            log.warn("Unable to locate SITE information for SITE CODE: " + siteCode);
            return;
        }
        String lab = site.getLab();
        lab = lab.isEmpty() ? siteCode : lab;

        try {
            email.setFrom(EMAIL_FROM);
            email.setSubject("DOE CODE Record " + md.getWorkflowStatus().toString());
            email.addTo(EMAIL_SUBMISSION);

            String softwareTitle = md.getSoftwareTitle().replaceAll("^\\h+|\\h+$", "");

            StringBuilder msg = new StringBuilder();
            msg.append("<html>");
            msg.append("A new DOE CODE record has been ").append(md.getWorkflowStatus()).append(" for ").append(lab)
                    .append(" and is awaiting approval:");

            msg.append("<P>Code ID: ").append(md.getCodeId());
            msg.append("<BR>Software Title: ").append(softwareTitle);
            msg.append("</html>");

            email.setHtmlMsg(msg.toString());

            email.send();
        } catch (EmailException e) {
            log.error("Failed to send submission/announcement notification message for #" + md.getCodeId());
            log.error("Message: " + e.getMessage());
        }
    }

    /**
     * Send an email notification on APPROVAL of DOE CODE records.
     *
     * @param md the METADATA to send notification for
     */
    private static void sendApprovalNotification(DOECodeMetadata md) {
        HtmlEmail email = new HtmlEmail();
        email.setCharset(org.apache.commons.mail.EmailConstants.UTF_8);
        email.setHostName(EMAIL_HOST);

        // if HOST or record OWNER or PROJECT MANAGER NAME isn't set, cannot send
        if (StringUtils.isEmpty(EMAIL_HOST) || null == md || StringUtils.isEmpty(md.getOwner())
                || StringUtils.isEmpty(PM_NAME))
            return;
        // only has meaning for APPROVED records
        if (!Status.Approved.equals(md.getWorkflowStatus()))
            return;

        try {
            // get the OWNER information
            User owner = UserServices.findUserByEmail(md.getOwner());
            if (null == owner) {
                log.warn("Unable to locate USER information for Code ID: " + md.getCodeId());
                return;
            }

            Long codeId = md.getCodeId();

            // lookup previous Snapshot status info for item
            EntityManager em = DoeServletContextListener.createEntityManager();
            TypedQuery<MetadataSnapshot> querySnapshot = em
                    .createNamedQuery("MetadataSnapshot.findByCodeIdLastNotStatus", MetadataSnapshot.class)
                    .setParameter("status", DOECodeMetadata.Status.Approved).setParameter("codeId", codeId);

            String lastApprovalFor = "submitted/announced";
            List<MetadataSnapshot> results = querySnapshot.setMaxResults(1).getResultList();
            for (MetadataSnapshot ms : results) {
                lastApprovalFor = ms.getSnapshotKey().getSnapshotStatus().toString().toLowerCase();
            }

            String softwareTitle = md.getSoftwareTitle().replaceAll("^\\h+|\\h+$", "");

            email.setFrom(EMAIL_FROM);
            email.setSubject("Approved -- DOE CODE ID: " + codeId + ", " + softwareTitle);
            email.addTo(md.getOwner());

            // if email is provided, BCC the Project Manager
            if (!StringUtils.isEmpty(PM_EMAIL))
                email.addBcc(PM_EMAIL, PM_NAME);

            StringBuilder msg = new StringBuilder();

            msg.append("<html>");
            msg.append("Dear ").append(owner.getFirstName()).append(" ").append(owner.getLastName()).append(":");

            msg.append("<P>Thank you -- your ").append(lastApprovalFor).append(" project, DOE CODE ID: <a href=\"")
                    .append(SITE_URL).append("/biblio/").append(codeId).append("\">").append(codeId)
                    .append("</a>, has been approved.  It is now <a href=\"").append(SITE_URL)
                    .append("\">searchable</a> in DOE CODE by, for example, title or CODE ID #.</P>");

            // OMIT the following for BUSINESS TYPE software, or last ANNOUNCED software
            if (!DOECodeMetadata.Type.B.equals(md.getSoftwareType())
                    && !lastApprovalFor.equalsIgnoreCase("announced")) {
                msg.append(
                        "<P>You may need to continue editing your project to announce it to the Department of Energy ")
                        .append("to ensure announcement and dissemination in accordance with DOE statutory responsibilities. For more information please see ")
                        .append("<a href=\"").append(SITE_URL)
                        .append("/faq#what-does-it-mean-to-announce\">What does it mean to announce scientific code to DOE CODE?</a></P>");
            }
            msg.append(
                    "<P>If you have questions such as What are the benefits of getting a DOI for code or software?, see the ")
                    .append("<a href=\"").append(SITE_URL).append("/faq\">DOE CODE FAQs</a>.</P>");
            msg.append(
                    "<P>If we can be of assistance, please do not hesitate to <a href=\"mailto:doecode@osti.gov\">Contact Us</a>.</P>");
            msg.append("<P>Sincerely,</P>");
            msg.append("<P>").append(PM_NAME).append("<BR/>Product Manager for DOE CODE<BR/>USDOE/OSTI</P>");

            msg.append("</html>");

            email.setHtmlMsg(msg.toString());

            email.send();
        } catch (EmailException e) {
            log.error("Unable to send APPROVAL notification for #" + md.getCodeId());
            log.error("Message: " + e.getMessage());
        }
    }

    /**
     * Send a POC email notification on SUBMISSION/APPROVAL of DOE CODE records.
     *
     * @param md the METADATA to send notification for
     */
    private static void sendPOCNotification(DOECodeMetadata md) {
        // if HOST or MD or PROJECT MANAGER NAME isn't set, cannot send
        if (StringUtils.isEmpty(EMAIL_HOST) || null == md || StringUtils.isEmpty(PM_NAME))
            return;

        Long codeId = md.getCodeId();
        String siteCode = md.getSiteOwnershipCode();
        Status workflowStatus = md.getWorkflowStatus();

        // if SITE OWNERSHIP isn't set, cannot send
        if (StringUtils.isEmpty(siteCode))
            return;

        // only applicable to APPROVED records
        if (!Status.Approved.equals(workflowStatus))
            return;

        // get the SITE information
        Site site = SiteServices.findSiteBySiteCode(siteCode);
        if (null == site) {
            log.warn("Unable to locate SITE information for SITE CODE: " + siteCode);
            return;
        }

        // lookup previous Snapshot status info for item
        EntityManager em = DoeServletContextListener.createEntityManager();
        TypedQuery<MetadataSnapshot> querySnapshot = em
                .createNamedQuery("MetadataSnapshot.findByCodeIdLastNotStatus", MetadataSnapshot.class)
                .setParameter("status", DOECodeMetadata.Status.Approved).setParameter("codeId", codeId);

        String lastApprovalFor = "submitted/announced";
        List<MetadataSnapshot> results = querySnapshot.setMaxResults(1).getResultList();
        for (MetadataSnapshot ms : results) {
            lastApprovalFor = ms.getSnapshotKey().getSnapshotStatus().toString().toLowerCase();
        }

        List<String> emails = site.getPocEmails();

        // if POC is setup
        if (emails != null && !emails.isEmpty()) {
            try {
                HtmlEmail email = new HtmlEmail();
                email.setCharset(org.apache.commons.mail.EmailConstants.UTF_8);
                email.setHostName(EMAIL_HOST);

                String lab = site.getLab();
                lab = lab.isEmpty() ? siteCode : lab;

                String softwareTitle = md.getSoftwareTitle().replaceAll("^\\h+|\\h+$", "");

                email.setFrom(EMAIL_FROM);
                email.setSubject("POC Notification -- " + workflowStatus + " -- DOE CODE ID: " + codeId + ", "
                        + softwareTitle);

                for (String pocEmail : emails)
                    email.addTo(pocEmail);

                // if email is provided, BCC the Project Manager
                if (!StringUtils.isEmpty(PM_EMAIL))
                    email.addBcc(PM_EMAIL, PM_NAME);

                StringBuilder msg = new StringBuilder();

                msg.append("<html>");
                msg.append("Dear Sir or Madam:");

                String biblioLink = SITE_URL + "/biblio/" + codeId;

                msg.append("<p>As a point of contact for ").append(lab)
                        .append(", we wanted to inform you that a software project, titled ").append(softwareTitle)
                        .append(", associated with your organization was ").append(lastApprovalFor)
                        .append(" to DOE CODE and assigned DOE CODE ID: ").append(codeId)
                        .append(".  This project record is discoverable in <a href=\"").append(SITE_URL)
                        .append("\">DOE CODE</a>, e.g. searching by the project title or DOE CODE ID #, and can be found here: <a href=\"")
                        .append(biblioLink).append("\">").append(biblioLink).append("</a></p>");

                msg.append(
                        "<p>If you have any questions, please do not hesitate to <a href=\"mailto:doecode@osti.gov\">Contact Us</a>.</p>");
                msg.append("<p>Sincerely,</p>");
                msg.append("<p>").append(PM_NAME).append("<br/>Product Manager for DOE CODE<br/>USDOE/OSTI</p>");

                msg.append("</html>");

                email.setHtmlMsg(msg.toString());

                email.send();
            } catch (EmailException e) {
                log.error("Unable to send POC notification to " + Arrays.toString(emails.toArray()) + " for #"
                        + md.getCodeId());
                log.error("Message: " + e.getMessage());
            }
        }
    }

    /**
     * As needed, Create/Update OSTI Hosted GitLab projects.
     *
     * @param md the METADATA to process GitLab for
     */
    private static void processOSTIGitLab(DOECodeMetadata md) throws Exception {
        try {
            // only process OSTI Hosted type
            if (!DOECodeMetadata.Accessibility.CO.equals(md.getAccessibility()))
                return;

            String fileName = md.getFileName();

            String codeId = String.valueOf(md.getCodeId());
            java.nio.file.Path uploadedFile = Paths.get(FILE_UPLOADS, String.valueOf(codeId), fileName);

            // if no file was uploaded, fail
            if (!Files.exists(uploadedFile))
                throw new Exception("File not found in Uploads directory! [" + uploadedFile.toString() + "]");

            // convert to base64 for GitLab
            String base64Content = convertBase64(new FileInputStream(uploadedFile.toString()));

            if (base64Content == null)
                throw new Exception("Base 64 Content required for OSTI Hosted project!");

            String projectName = "dc-" + md.getCodeId();
            String hostedFolder = "hosted_files";

            String uploadFile = hostedFolder + "/" + fileName;

            GitLabAPI glApi = new GitLabAPI();
            glApi.setProjectName(projectName);

            // check existance of project
            Project project = glApi.fetchProject();

            Commit commit = new Commit();
            if (project == null) {
                // create new project, if none exists
                project = glApi.createProject(md);

                commit.setBranch(glApi.getBranch());
                commit.setCommitMessage("Adding Hosted Files: " + fileName);
                commit.addBase64ActionByValues("create", uploadFile, base64Content);
            } else {
                // edit project, if one already exists
                project = glApi.updateProject(md);

                // for each file in the hosted folder, check against submitted files and process as needed
                GitLabFile[] files = glApi.fetchTree(hostedFolder);

                int adds = 0;
                int updates = 0;
                int deletes = 0;

                if (files != null) {
                    for (GitLabFile f : files) {
                        if (!f.getType().equalsIgnoreCase("tree")) {
                            if (f.getPath().equals(uploadFile)) {
                                // update, if filename exists already
                                commit.addBase64ActionByValues("update", uploadFile, base64Content);

                                updates++;
                            } else {
                                // delete, if file is not being submitted
                                commit.addBase64ActionByValues("delete", f.getPath(), null);

                                deletes++;
                            }
                        }
                    }
                }

                // right now there is only ever one file, so if we did not updated anything, we need to create it
                if (updates == 0) {
                    // create, if there was no matching file to update
                    commit.addBase64ActionByValues("create", uploadFile, base64Content);

                    adds++;
                }

                String prefix;
                String detail = "\"" + fileName + "\"";
                if (adds == 1 && updates == 0 && deletes == 0) {
                    prefix = "Adding";
                } else if (adds == 0 && updates == 1 && deletes == 0) {
                    prefix = "Updating";
                } else {
                    prefix = "Modifying";

                    String tmp = (adds == 1) ? "\"" + fileName + "\"" : String.valueOf(adds);
                    detail = (adds > 0 ? "Added " + tmp : "");

                    tmp = (updates == 1) ? "\"" + fileName + "\"" : String.valueOf(updates);
                    detail += (updates > 0 ? (StringUtils.isBlank(detail) ? "" : ", ") + "Updated " + tmp : "");

                    detail += (deletes > 0 ? (StringUtils.isBlank(detail) ? "" : ", ") + "Deleted " + deletes : "");
                }

                commit.setBranch(project.getDefaultBranch());
                commit.setCommitMessage(prefix + " Hosted Files: " + detail);
            }

            // override repo link, no matter what
            if (!StringUtils.isBlank(project.getWebUrl()))
                md.setRepositoryLink(project.getWebUrl());
            else
                throw new Exception("Unable to determine GitLab Repository URL!");

            // commit file actions, as needed
            glApi.commitFiles(commit);

        } catch (Exception e) {
            // replace non-obvious 'name' with 'software_title' for user clarity
            String eMsg = e.getMessage();
            eMsg = eMsg.replaceFirst("'name'", "'software_title'");
            throw new Exception(eMsg);
        }
    }

    /**
     * As needed, move Container Uploads to download location.
     *
     * @param md the METADATA to process Container Approval for
     */
    private static void approveContainerUpload(DOECodeMetadata md) throws IOException {
        String containerName = md.getContainerName();

        // if nothing to move, return
        if (StringUtils.isBlank(containerName))
            return;

        String codeId = String.valueOf(md.getCodeId());
        java.nio.file.Path uploadedFile = Paths.get(CONTAINER_UPLOADS, String.valueOf(codeId), containerName);
        java.nio.file.Path approvedFile = Paths.get(CONTAINER_UPLOADS_APPROVED, String.valueOf(codeId),
                containerName);

        // if file already moved to approval, skip
        if (!Files.exists(uploadedFile) && Files.exists(approvedFile))
            return;

        // if file is missing, fail
        if (!Files.exists(uploadedFile))
            throw new IOException("Container not found in containers directory during Approval! ["
                    + uploadedFile.toString() + "]");

        // make intervening folders if needed
        Files.createDirectories(approvedFile.getParent());

        try {
            Files.move(uploadedFile, approvedFile, REPLACE_EXISTING, ATOMIC_MOVE);
        } catch (IOException e) {
            String eMsg = "Failed to move Container during Approval: " + e.getMessage();
            throw new IOException(eMsg);
        }
    }
}