edu.harvard.iq.dataverse.api.imports.ImportServiceBean.java Source code

Java tutorial

Introduction

Here is the source code for edu.harvard.iq.dataverse.api.imports.ImportServiceBean.java

Source

/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package edu.harvard.iq.dataverse.api.imports;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import edu.harvard.iq.dataverse.DataFile;
import edu.harvard.iq.dataverse.Dataset;
import edu.harvard.iq.dataverse.DatasetField;
import edu.harvard.iq.dataverse.DatasetFieldConstant;
import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
import edu.harvard.iq.dataverse.DatasetFieldType;
import edu.harvard.iq.dataverse.DatasetFieldValue;
import edu.harvard.iq.dataverse.DatasetServiceBean;
import edu.harvard.iq.dataverse.DatasetVersion;
import edu.harvard.iq.dataverse.Dataverse;
import edu.harvard.iq.dataverse.DataverseContact;
import edu.harvard.iq.dataverse.DataverseServiceBean;
import edu.harvard.iq.dataverse.EjbDataverseEngine;
import edu.harvard.iq.dataverse.ForeignMetadataFormatMapping;
import edu.harvard.iq.dataverse.MetadataBlockServiceBean;
import edu.harvard.iq.dataverse.api.dto.DatasetDTO;
import edu.harvard.iq.dataverse.api.imports.ImportUtil.ImportType;
import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
import edu.harvard.iq.dataverse.authorization.users.User;
import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
import edu.harvard.iq.dataverse.engine.command.impl.CreateDatasetCommand;
import edu.harvard.iq.dataverse.engine.command.impl.CreateDatasetVersionCommand;
import edu.harvard.iq.dataverse.engine.command.impl.CreateDataverseCommand;
import edu.harvard.iq.dataverse.engine.command.impl.DestroyDatasetCommand;
import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
import edu.harvard.iq.dataverse.search.IndexServiceBean;
import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
import edu.harvard.iq.dataverse.util.json.JsonParseException;
import edu.harvard.iq.dataverse.util.json.JsonParser;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringReader;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.logging.Formatter;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
import javax.ejb.EJB;
import javax.ejb.EJBException;
import javax.ejb.Stateless;
import javax.ejb.TransactionAttribute;
import javax.ejb.TransactionAttributeType;
import static javax.ejb.TransactionAttributeType.REQUIRES_NEW;
import javax.json.Json;
import javax.json.JsonObject;
import javax.json.JsonObjectBuilder;
import javax.json.JsonReader;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.servlet.http.HttpServletRequest;
import javax.validation.ConstraintViolation;
import javax.validation.ConstraintViolationException;
import javax.validation.Validation;
import javax.validation.Validator;
import javax.validation.ValidatorFactory;
import javax.ws.rs.core.Context;
import javax.xml.stream.XMLStreamException;
import org.apache.commons.lang.StringUtils;

/**
 *
 * @author ellenk
 * TODO: Why does this bean not extend AbstractApiBean?
 */
@Stateless
public class ImportServiceBean {
    @PersistenceContext(unitName = "VDCNet-ejbPU")
    private EntityManager em;

    private static final Logger logger = Logger.getLogger(ImportServiceBean.class.getCanonicalName());

    @EJB
    protected EjbDataverseEngine engineSvc;
    @EJB
    DatasetServiceBean datasetService;
    @EJB
    DataverseServiceBean dataverseService;
    @EJB
    DatasetFieldServiceBean datasetfieldService;

    @EJB
    MetadataBlockServiceBean metadataBlockService;
    @EJB
    SettingsServiceBean settingsService;

    @EJB
    ImportDDIServiceBean importDDIService;
    @EJB
    ImportGenericServiceBean importGenericService;

    @EJB
    IndexServiceBean indexService;

    /**
     * This is just a convenience method, for testing migration.  It creates 
     * a dummy dataverse with the directory name as dataverse name & alias.
     * @param dvName
     * @param dataverseRequest
     * @return
     * @throws ImportException 
     */
    @TransactionAttribute(REQUIRES_NEW)
    public Dataverse createDataverse(String dvName, DataverseRequest dataverseRequest) throws ImportException {
        Dataverse d = new Dataverse();
        Dataverse root = dataverseService.findByAlias("root");
        d.setOwner(root);
        d.setAlias(dvName);
        d.setName(dvName);
        d.setAffiliation("affiliation");
        d.setPermissionRoot(false);
        d.setDescription("description");
        d.setDataverseType(Dataverse.DataverseType.RESEARCHERS);
        DataverseContact dc = new DataverseContact();
        dc.setContactEmail("pete@mailinator.com");
        ArrayList<DataverseContact> dcList = new ArrayList<>();
        dcList.add(dc);
        d.setDataverseContacts(dcList);
        try {
            d = engineSvc.submit(new CreateDataverseCommand(d, dataverseRequest, null, null));
        } catch (EJBException ex) {
            Throwable cause = ex;
            StringBuilder sb = new StringBuilder();
            sb.append("Error creating dataverse.");
            while (cause.getCause() != null) {
                cause = cause.getCause();
                if (cause instanceof ConstraintViolationException) {
                    ConstraintViolationException constraintViolationException = (ConstraintViolationException) cause;
                    for (ConstraintViolation<?> violation : constraintViolationException
                            .getConstraintViolations()) {
                        sb.append(" Invalid value: <<<").append(violation.getInvalidValue()).append(">>> for ")
                                .append(violation.getPropertyPath()).append(" at ").append(violation.getLeafBean())
                                .append(" - ").append(violation.getMessage());
                    }
                }
            }
            logger.log(Level.SEVERE, sb.toString());
            System.out.println("Error creating dataverse: " + sb.toString());
            throw new ImportException(sb.toString());
        } catch (CommandException e) {
            throw new ImportException(e.getMessage());
        }
        return d;

    }

    @TransactionAttribute(REQUIRES_NEW)
    public JsonObjectBuilder handleFile(DataverseRequest dataverseRequest, Dataverse owner, File file,
            ImportType importType, PrintWriter validationLog, PrintWriter cleanupLog)
            throws ImportException, IOException {

        System.out.println("handling file: " + file.getAbsolutePath());
        String ddiXMLToParse;
        try {
            ddiXMLToParse = new String(Files.readAllBytes(file.toPath()));
            JsonObjectBuilder status = doImport(dataverseRequest, owner, ddiXMLToParse,
                    file.getParentFile().getName() + "/" + file.getName(), importType, cleanupLog);
            status.add("file", file.getName());
            logger.log(Level.INFO, "completed doImport {0}/{1}",
                    new Object[] { file.getParentFile().getName(), file.getName() });
            return status;
        } catch (ImportException ex) {
            String msg = "Import Exception processing file " + file.getParentFile().getName() + "/" + file.getName()
                    + ", msg:" + ex.getMessage();
            logger.info(msg);
            if (validationLog != null) {
                validationLog.println(msg);
            }
            return Json.createObjectBuilder().add("message", "Import Exception processing file "
                    + file.getParentFile().getName() + "/" + file.getName() + ", msg:" + ex.getMessage());
        } catch (IOException e) {
            Throwable causedBy = e.getCause();
            while (causedBy != null && causedBy.getCause() != null) {
                causedBy = causedBy.getCause();
            }
            String stackLine = "";
            if (causedBy != null && causedBy.getStackTrace() != null && causedBy.getStackTrace().length > 0) {
                stackLine = causedBy.getStackTrace()[0].toString();
            }
            String msg = "Unexpected Error in handleFile(), file:" + file.getParentFile().getName() + "/"
                    + file.getName();
            if (e.getMessage() != null) {
                msg += "message: " + e.getMessage();
            }
            msg += ", caused by: " + causedBy;
            if (causedBy != null && causedBy.getMessage() != null) {
                msg += ", caused by message: " + causedBy.getMessage();
            }
            msg += " at line: " + stackLine;

            validationLog.println(msg);
            e.printStackTrace();

            return Json.createObjectBuilder().add("message", "Unexpected Exception processing file "
                    + file.getParentFile().getName() + "/" + file.getName() + ", msg:" + e.getMessage());

        }
    }

    @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
    public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, HarvestingClient harvestingClient,
            String harvestIdentifier, String metadataFormat, File metadataFile, PrintWriter cleanupLog)
            throws ImportException, IOException {
        if (harvestingClient == null || harvestingClient.getDataverse() == null) {
            throw new ImportException(
                    "importHarvestedDataset called wiht a null harvestingClient, or an invalid harvestingClient.");
        }
        Dataverse owner = harvestingClient.getDataverse();
        Dataset importedDataset = null;

        DatasetDTO dsDTO = null;
        String json = null;

        // TODO: 
        // At the moment (4.5; the first official "export/harvest release"), there
        // are 3 supported metadata formats: DDI, DC and native Dataverse metadata 
        // encoded in JSON. The 2 XML formats are handled by custom implementations;
        // each of the 2 implementations uses its own parsing approach. (see the 
        // ImportDDIServiceBean and ImportGenerciServiceBean for details). 
        // TODO: Need to create a system of standardized import plugins - similar to Stephen
        // Kraffmiller's export modules; replace the logic below with clean
        // programmatic lookup of the import plugin needed. 

        if ("ddi".equalsIgnoreCase(metadataFormat) || "oai_ddi".equals(metadataFormat)
                || metadataFormat.toLowerCase().matches("^oai_ddi.*")) {
            try {
                String xmlToParse = new String(Files.readAllBytes(metadataFile.toPath()));
                // TODO: 
                // import type should be configurable - it should be possible to 
                // select whether you want to harvest with or without files, 
                // ImportType.HARVEST vs. ImportType.HARVEST_WITH_FILES
                logger.fine("importing DDI " + metadataFile.getAbsolutePath());
                dsDTO = importDDIService.doImport(ImportType.HARVEST_WITH_FILES, xmlToParse);
            } catch (IOException | XMLStreamException | ImportException e) {
                throw new ImportException(
                        "Failed to process DDI XML record: " + e.getClass() + " (" + e.getMessage() + ")");
            }
        } else if ("dc".equalsIgnoreCase(metadataFormat) || "oai_dc".equals(metadataFormat)) {
            logger.fine("importing DC " + metadataFile.getAbsolutePath());
            try {
                String xmlToParse = new String(Files.readAllBytes(metadataFile.toPath()));
                dsDTO = importGenericService.processOAIDCxml(xmlToParse);
            } catch (IOException | XMLStreamException e) {
                throw new ImportException(
                        "Failed to process Dublin Core XML record: " + e.getClass() + " (" + e.getMessage() + ")");
            }
        } else if ("dataverse_json".equals(metadataFormat)) {
            // This is Dataverse metadata already formatted in JSON. 
            // Simply read it into a string, and pass to the final import further down:
            logger.fine(
                    "Attempting to import custom dataverse metadata from file " + metadataFile.getAbsolutePath());
            json = new String(Files.readAllBytes(metadataFile.toPath()));
        } else {
            throw new ImportException("Unsupported import metadata format: " + metadataFormat);
        }

        if (json == null) {
            if (dsDTO != null) {
                // convert DTO to Json, 
                Gson gson = new GsonBuilder().setPrettyPrinting().create();
                json = gson.toJson(dsDTO);
                logger.fine("JSON produced for the metadata harvested: " + json);
            } else {
                throw new ImportException(
                        "Failed to transform XML metadata format " + metadataFormat + " into a DatasetDTO");
            }
        }

        JsonReader jsonReader = Json.createReader(new StringReader(json));
        JsonObject obj = jsonReader.readObject();
        //and call parse Json to read it into a dataset   
        try {
            JsonParser parser = new JsonParser(datasetfieldService, metadataBlockService, settingsService);
            parser.setLenient(true);
            Dataset ds = parser.parseDataset(obj);

            // For ImportType.NEW, if the metadata contains a global identifier, and it's not a protocol
            // we support, it should be rejected.
            // (TODO: ! - add some way of keeping track of supported protocols!)
            //if (ds.getGlobalId() != null && !ds.getProtocol().equals(settingsService.getValueForKey(SettingsServiceBean.Key.Protocol, ""))) {
            //    throw new ImportException("Could not register id " + ds.getGlobalId() + ", protocol not supported");
            //}
            ds.setOwner(owner);
            ds.getLatestVersion().setDatasetFields(ds.getLatestVersion().initDatasetFields());

            // Check data against required contraints
            List<ConstraintViolation<DatasetField>> violations = ds.getVersions().get(0).validateRequired();
            if (!violations.isEmpty()) {
                // For migration and harvest, add NA for missing required values
                for (ConstraintViolation<DatasetField> v : violations) {
                    DatasetField f = v.getRootBean();
                    f.setSingleValue(DatasetField.NA_VALUE);
                }
            }

            // Check data against validation constraints
            // If we are migrating and "scrub migration data" is true we attempt to fix invalid data
            // if the fix fails stop processing of this file by throwing exception
            Set<ConstraintViolation> invalidViolations = ds.getVersions().get(0).validate();
            ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
            Validator validator = factory.getValidator();
            if (!invalidViolations.isEmpty()) {
                for (ConstraintViolation<DatasetFieldValue> v : invalidViolations) {
                    DatasetFieldValue f = v.getRootBean();
                    boolean fixed = false;
                    boolean converted = false;
                    // TODO: Is this scrubbing something we want to continue doing? 
                    if (settingsService.isTrueForKey(SettingsServiceBean.Key.ScrubMigrationData, false)) {
                        fixed = processMigrationValidationError(f, cleanupLog, metadataFile.getName());
                        converted = true;
                        if (fixed) {
                            Set<ConstraintViolation<DatasetFieldValue>> scrubbedViolations = validator.validate(f);
                            if (!scrubbedViolations.isEmpty()) {
                                fixed = false;
                            }
                        }
                    }
                    if (!fixed) {
                        String msg = "Data modified - File: " + metadataFile.getName() + "; Field: "
                                + f.getDatasetField().getDatasetFieldType().getDisplayName() + "; "
                                + "Invalid value:  '" + f.getValue() + "'" + " Converted Value:'"
                                + DatasetField.NA_VALUE + "'";
                        cleanupLog.println(msg);
                        f.setValue(DatasetField.NA_VALUE);

                    }
                }
            }

            // A Global ID is required, in order for us to be able to harvest and import
            // this dataset:
            if (StringUtils.isEmpty(ds.getGlobalId())) {
                throw new ImportException("The harvested metadata record with the OAI server identifier "
                        + harvestIdentifier
                        + " does not contain a global unique identifier that we could recognize, skipping.");
            }

            ds.setHarvestedFrom(harvestingClient);
            ds.setHarvestIdentifier(harvestIdentifier);

            Dataset existingDs = datasetService.findByGlobalId(ds.getGlobalId());

            if (existingDs != null) {
                // If this dataset already exists IN ANOTHER DATAVERSE
                // we are just going to skip it!
                if (existingDs.getOwner() != null && !owner.getId().equals(existingDs.getOwner().getId())) {
                    throw new ImportException("The dataset with the global id " + ds.getGlobalId()
                            + " already exists, in the dataverse " + existingDs.getOwner().getAlias()
                            + ", skipping.");
                }
                // And if we already have a dataset with this same id, in this same
                // dataverse, but it is  LOCAL dataset (can happen!), we're going to 
                // skip it also: 
                if (!existingDs.isHarvested()) {
                    throw new ImportException("A LOCAL dataset with the global id " + ds.getGlobalId()
                            + " already exists in this dataverse; skipping.");
                }
                // For harvested datasets, there should always only be one version.
                // We will replace the current version with the imported version.
                if (existingDs.getVersions().size() != 1) {
                    throw new ImportException("Error importing Harvested Dataset, existing dataset has "
                            + existingDs.getVersions().size() + " versions");
                }
                // Purge all the SOLR documents associated with this client from the 
                // index server: 
                indexService.deleteHarvestedDocuments(existingDs);
                // files from harvested datasets are removed unceremoniously, 
                // directly in the database. no need to bother calling the 
                // DeleteFileCommand on them.
                for (DataFile harvestedFile : existingDs.getFiles()) {
                    DataFile merged = em.merge(harvestedFile);
                    em.remove(merged);
                    harvestedFile = null;
                }
                // TODO: 
                // Verify what happens with the indexed files in SOLR? 
                // are they going to be overwritten by the reindexing of the dataset?
                existingDs.setFiles(null);
                Dataset merged = em.merge(existingDs);
                engineSvc.submit(new DestroyDatasetCommand(merged, dataverseRequest));
                importedDataset = engineSvc
                        .submit(new CreateDatasetCommand(ds, dataverseRequest, false, ImportType.HARVEST));

            } else {
                importedDataset = engineSvc
                        .submit(new CreateDatasetCommand(ds, dataverseRequest, false, ImportType.HARVEST));
            }

        } catch (JsonParseException | ImportException | CommandException ex) {
            logger.fine("Failed to import harvested dataset: " + ex.getClass() + ": " + ex.getMessage());
            FileOutputStream savedJsonFileStream = new FileOutputStream(
                    new File(metadataFile.getAbsolutePath() + ".json"));
            byte[] jsonBytes = json.getBytes();
            int i = 0;
            while (i < jsonBytes.length) {
                int chunkSize = i + 8192 <= jsonBytes.length ? 8192 : jsonBytes.length - i;
                savedJsonFileStream.write(jsonBytes, i, chunkSize);
                i += chunkSize;
                savedJsonFileStream.flush();
            }
            savedJsonFileStream.close();
            logger.info("JSON produced saved in " + metadataFile.getAbsolutePath() + ".json");
            throw new ImportException(
                    "Failed to import harvested dataset: " + ex.getClass() + " (" + ex.getMessage() + ")", ex);
        }
        return importedDataset;
    }

    public JsonObjectBuilder doImport(DataverseRequest dataverseRequest, Dataverse owner, String xmlToParse,
            String fileName, ImportType importType, PrintWriter cleanupLog) throws ImportException, IOException {

        String status = "";
        Long createdId = null;
        DatasetDTO dsDTO = null;
        try {

            dsDTO = importDDIService.doImport(importType, xmlToParse);
        } catch (XMLStreamException e) {
            throw new ImportException("XMLStreamException" + e);
        }
        // convert DTO to Json, 
        Gson gson = new GsonBuilder().setPrettyPrinting().create();
        String json = gson.toJson(dsDTO);
        JsonReader jsonReader = Json.createReader(new StringReader(json));
        JsonObject obj = jsonReader.readObject();
        //and call parse Json to read it into a dataset   
        try {
            JsonParser parser = new JsonParser(datasetfieldService, metadataBlockService, settingsService);
            parser.setLenient(!importType.equals(ImportType.NEW));
            Dataset ds = parser.parseDataset(obj);

            // For ImportType.NEW, if the user supplies a global identifier, and it's not a protocol
            // we support, it will be rejected.
            if (importType.equals(ImportType.NEW)) {
                if (ds.getGlobalId() != null && !ds.getProtocol()
                        .equals(settingsService.getValueForKey(SettingsServiceBean.Key.Protocol, ""))) {
                    throw new ImportException(
                            "Could not register id " + ds.getGlobalId() + ", protocol not supported");
                }
            }

            ds.setOwner(owner);
            ds.getLatestVersion().setDatasetFields(ds.getLatestVersion().initDatasetFields());

            // Check data against required contraints
            List<ConstraintViolation<DatasetField>> violations = ds.getVersions().get(0).validateRequired();
            if (!violations.isEmpty()) {
                if (importType.equals(ImportType.MIGRATION) || importType.equals(ImportType.HARVEST)) {
                    // For migration and harvest, add NA for missing required values
                    for (ConstraintViolation<DatasetField> v : violations) {
                        DatasetField f = v.getRootBean();
                        f.setSingleValue(DatasetField.NA_VALUE);
                    }
                } else {
                    // when importing a new dataset, the import will fail
                    // if required values are missing.
                    String errMsg = "Error importing data:";
                    for (ConstraintViolation<DatasetField> v : violations) {
                        errMsg += " " + v.getMessage();
                    }
                    throw new ImportException(errMsg);
                }
            }

            // Check data against validation constraints
            // If we are migrating and "scrub migration data" is true we attempt to fix invalid data
            // if the fix fails stop processing of this file by throwing exception
            Set<ConstraintViolation> invalidViolations = ds.getVersions().get(0).validate();
            ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
            Validator validator = factory.getValidator();
            if (!invalidViolations.isEmpty()) {
                for (ConstraintViolation<DatasetFieldValue> v : invalidViolations) {
                    DatasetFieldValue f = v.getRootBean();
                    boolean fixed = false;
                    boolean converted = false;
                    if ((importType.equals(ImportType.MIGRATION) || importType.equals(ImportType.HARVEST))
                            && settingsService.isTrueForKey(SettingsServiceBean.Key.ScrubMigrationData, false)) {
                        fixed = processMigrationValidationError(f, cleanupLog, fileName);
                        converted = true;
                        if (fixed) {
                            Set<ConstraintViolation<DatasetFieldValue>> scrubbedViolations = validator.validate(f);
                            if (!scrubbedViolations.isEmpty()) {
                                fixed = false;
                            }
                        }
                    }
                    if (!fixed) {
                        if (importType.equals(ImportType.HARVEST)) {
                            String msg = "Data modified - File: " + fileName + "; Field: "
                                    + f.getDatasetField().getDatasetFieldType().getDisplayName() + "; "
                                    + "Invalid value:  '" + f.getValue() + "'" + " Converted Value:'"
                                    + DatasetField.NA_VALUE + "'";
                            cleanupLog.println(msg);
                            f.setValue(DatasetField.NA_VALUE);

                        } else {
                            String msg = " Validation error for ";
                            if (converted) {
                                msg += "converted ";
                            }
                            msg += "value: " + f.getValue() + ", " + f.getValidationMessage();
                            throw new ImportException(msg);
                        }
                    }
                }
            }

            Dataset existingDs = datasetService.findByGlobalId(ds.getGlobalId());

            if (existingDs != null) {
                if (importType.equals(ImportType.HARVEST)) {
                    // For harvested datasets, there should always only be one version.
                    // We will replace the current version with the imported version.
                    if (existingDs.getVersions().size() != 1) {
                        throw new ImportException("Error importing Harvested Dataset, existing dataset has "
                                + existingDs.getVersions().size() + " versions");
                    }
                    engineSvc.submit(new DestroyDatasetCommand(existingDs, dataverseRequest));
                    Dataset managedDs = engineSvc
                            .submit(new CreateDatasetCommand(ds, dataverseRequest, false, importType));
                    status = " updated dataset, id=" + managedDs.getId() + ".";
                } else {
                    // If we are adding a new version to an existing dataset,
                    // check that the version number isn't already in the dataset
                    for (DatasetVersion dsv : existingDs.getVersions()) {
                        if (dsv.getVersionNumber().equals(ds.getLatestVersion().getVersionNumber())) {
                            throw new ImportException("VersionNumber " + ds.getLatestVersion().getVersionNumber()
                                    + " already exists in dataset " + existingDs.getGlobalId());
                        }
                    }
                    DatasetVersion dsv = engineSvc.submit(
                            new CreateDatasetVersionCommand(dataverseRequest, existingDs, ds.getVersions().get(0)));
                    status = " created datasetVersion, for dataset " + dsv.getDataset().getGlobalId();
                    createdId = dsv.getId();
                }

            } else {
                Dataset managedDs = engineSvc
                        .submit(new CreateDatasetCommand(ds, dataverseRequest, false, importType));
                status = " created dataset, id=" + managedDs.getId() + ".";
                createdId = managedDs.getId();
            }

        } catch (JsonParseException ex) {
            logger.log(Level.INFO, "Error parsing datasetVersion: {0}", ex.getMessage());
            throw new ImportException("Error parsing datasetVersion: " + ex.getMessage(), ex);
        } catch (CommandException ex) {
            logger.log(Level.INFO, "Error excuting Create dataset command: {0}", ex.getMessage());
            throw new ImportException("Error excuting dataverse command: " + ex.getMessage(), ex);
        }
        return Json.createObjectBuilder().add("message", status);
    }

    private boolean processMigrationValidationError(DatasetFieldValue f, PrintWriter cleanupLog, String fileName) {
        if (f.getDatasetField().getDatasetFieldType().getName().equals(DatasetFieldConstant.datasetContactEmail)) {
            //Try to convert it based on the errors we've seen
            String convertedVal = convertInvalidEmail(f.getValue());
            if (!(convertedVal == null)) {
                String msg = "Data modified - File: " + fileName + "; Field: "
                        + f.getDatasetField().getDatasetFieldType().getDisplayName() + "; " + "Invalid value:  '"
                        + f.getValue() + "'" + " Converted Value:'" + convertedVal + "'";
                cleanupLog.println(msg);
                f.setValue(convertedVal);
                return true;
            }
            //if conversion fails set to NA
            String msg = "Data modified - File: " + fileName + "; Field: Dataset Contact Email; "
                    + "Invalid value: '" + f.getValue() + "'" + " Converted Value: 'NA'";
            cleanupLog.println(msg);
            f.setValue(DatasetField.NA_VALUE);
            return true;
        }
        if (f.getDatasetField().getDatasetFieldType().getName().equals(DatasetFieldConstant.producerURL)) {
            if (f.getValue().equals("PRODUCER URL")) {
                String msg = "Data modified - File: " + fileName + "; Field: Producer URL; " + "Invalid value: '"
                        + f.getValue() + "'" + " Converted Value: 'NA'";
                cleanupLog.println(msg);
                f.setValue(DatasetField.NA_VALUE);
                return true;
            }
        }
        if (f.getDatasetField().getDatasetFieldType().getFieldType().equals(DatasetFieldType.FieldType.DATE)) {
            if (f.getValue().toUpperCase().equals("YYYY-MM-DD")) {
                String msg = "Data modified - File: " + fileName + "; Field:"
                        + f.getDatasetField().getDatasetFieldType().getDisplayName() + "; " + "Invalid value: '"
                        + f.getValue() + "'" + " Converted Value: 'NA'";
                cleanupLog.println(msg);
                f.setValue(DatasetField.NA_VALUE);
                return true;
            }
            String convertedVal = convertInvalidDateString(f.getValue());
            if (!(convertedVal == null)) {
                String msg = "Data modified - File: " + fileName + "; Field: "
                        + f.getDatasetField().getDatasetFieldType().getDisplayName() + "" + " Converted Value:"
                        + convertedVal + "; Invalid value:  '" + f.getValue() + "'";
                cleanupLog.println(msg);
                f.setValue(convertedVal);
                return true;
            }
        }
        return false;
    }

    private String convertInvalidEmail(String inString) {
        //First we'll see if the invalid email is a comma delimited list of email addresses
        //if so we'll return the first one - maybe try to get them all at some point?
        if (inString.contains(",")) {
            String[] addresses = inString.split("\\,");
            return addresses[0];
        }

        //This works on the specific error we've seen where the user has put in a link for the email address
        //as in '<a href="IFPRI-Data@cgiar.org" > IFPRI-Data@cgiar.org</a>'
        //this returns the string between the first > and the second <
        if (inString.indexOf("<a", 0) > -1) {
            try {
                String eMailAddress = inString.substring(inString.indexOf(">", 0) + 1,
                        inString.indexOf("</a>", inString.indexOf(">", 0)));
                return eMailAddress.trim();
            } catch (Exception e) {
                return null;
            }
        }
        return null;
    }

    private String convertInvalidDateString(String inString) {

        //converts XXXX0000 to XXXX for date purposes
        if (inString.trim().length() == 8) {
            if (inString.trim().endsWith("0000")) {
                return inString.replace("0000", "").trim();
            }
        }

        //convert question marks to dashes and add brackets

        if (inString.contains("?")) {
            String testval = inString.replace("?", " ").replace("[", " ").replace("]", " ");
            if (StringUtils.isNumeric(testval.trim())) {
                switch (testval.trim().length()) {
                case 1:
                    return "[" + testval.trim() + "---?]";
                case 2:
                    return "[" + testval.trim() + "--?]";
                case 3:
                    return "[" + testval.trim() + "-?]";
                case 4:
                    return "[" + testval.trim() + "?]";
                case 8:
                    if (testval.trim().contains("0000")) {
                        return "[" + testval.trim().replace("0000", "") + "?]";
                    }
                }
            }
        }

        //Convert string months to numeric    

        if (inString.toUpperCase().contains("JANUARY")) {
            return inString.toUpperCase().replace("JANUARY", "").replace(",", "").trim() + "-01";
        }

        if (inString.toUpperCase().contains("FEBRUARY")) {
            return inString.toUpperCase().replace("FEBRUARY", "").replace(",", "").trim() + "-02";
        }

        if (inString.toUpperCase().contains("MARCH")) {
            return inString.toUpperCase().replace("MARCH", "").replace(",", "").trim() + "-03";
        }

        if (inString.toUpperCase().contains("APRIL")) {
            return inString.toUpperCase().replace("APRIL", "").replace(",", "").trim() + "-04";
        }

        if (inString.toUpperCase().contains("MAY")) {
            return inString.toUpperCase().replace("MAY", "").replace(",", "").trim() + "-05";
        }

        if (inString.toUpperCase().contains("JUNE")) {
            return inString.toUpperCase().replace("JUNE", "").replace(",", "").trim() + "-06";
        }

        if (inString.toUpperCase().contains("JULY")) {
            return inString.toUpperCase().replace("JULY", "").replace(",", "").trim() + "-07";
        }

        if (inString.toUpperCase().contains("AUGUST")) {
            return inString.toUpperCase().replace("AUGUST", "").replace(",", "").trim() + "-08";
        }

        if (inString.toUpperCase().contains("SEPTEMBER")) {
            return inString.toUpperCase().replace("SEPTEMBER", "").replace(",", "").trim() + "-09";
        }

        if (inString.toUpperCase().contains("OCTOBER")) {
            return inString.toUpperCase().replace("OCTOBER", "").replace(",", "").trim() + "-10";
        }

        if (inString.toUpperCase().contains("NOVEMBER")) {
            return inString.toUpperCase().replace("NOVEMBER", "").replace(",", "").trim() + "-11";
        }

        if (inString.toUpperCase().contains("DECEMBER")) {
            return inString.toUpperCase().replace("DECEMBER", "").replace(",", "").trim() + "-12";
        }

        return null;
    }

    private static class MyCustomFormatter extends Formatter {

        @Override

        public String format(LogRecord record) {

            StringBuffer sb = new StringBuffer();

            sb.append("Prefixn");

            sb.append(record.getMessage());

            sb.append("Suffixn");

            sb.append("n");

            return sb.toString();

        }

    }

}