Java tutorial
/******************************************************************************* * Educational Online Test Delivery System * Copyright (c) 2014 American Institutes for Research * * Distributed under the AIR Open Source License, Version 1.0 * See accompanying file AIR-License-1_0.txt or at * http://www.smarterapp.org/documents/American_Institutes_for_Research_Open_Source_Software_License.pdf ******************************************************************************/ package org.opentestsystem.authoring.testitembank.service.impl; import java.io.File; import java.util.List; import java.util.Map.Entry; import org.apache.commons.io.FileUtils; import org.joda.time.DateTime; import org.opentestsystem.authoring.testitembank.apipzip.domain.ApipItemContent; import org.opentestsystem.authoring.testitembank.domain.ImportFile; import org.opentestsystem.authoring.testitembank.domain.ImportSet; import org.opentestsystem.authoring.testitembank.domain.ImportStatus; import org.opentestsystem.authoring.testitembank.domain.Item; import org.opentestsystem.authoring.testitembank.domain.MnaAlertType; import org.opentestsystem.authoring.testitembank.exception.TestItemBankException; import org.opentestsystem.authoring.testitembank.persistence.GridFsRepository; import org.opentestsystem.authoring.testitembank.persistence.ImportSetRepository; import org.opentestsystem.authoring.testitembank.persistence.ItemRepository; import org.opentestsystem.authoring.testitembank.service.FileTransferService; import org.opentestsystem.authoring.testitembank.service.ImportSetService; import org.opentestsystem.authoring.testitembank.service.ItemMetadataService; import org.opentestsystem.authoring.testitembank.service.ZipInputFileExtractorService; import org.opentestsystem.authoring.testitembank.validation.ItemValidator; import org.opentestsystem.shared.mna.client.domain.MnaSeverity; import org.opentestsystem.shared.mna.client.service.AlertBeacon; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.dao.DuplicateKeyException; import org.springframework.scheduling.annotation.Async; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.springframework.validation.BindException; import org.springframework.validation.Errors; import org.springframework.validation.ObjectError; import com.mongodb.BasicDBObject; import com.mongodb.gridfs.GridFSFile; @Service public class ImportSetServiceImpl implements ImportSetService { private static final Logger LOGGER = LoggerFactory.getLogger(ImportSetServiceImpl.class); public static final String SFTP_TENANT_FOLDER_PREFIX = "tenant_"; @Value(value = "${tib.clean.files.age.days:0}") private Integer maxImportFileAge; @Value(value = "${tib.sftp.import.directory:}") private String tibImportDirectory; @Autowired private AlertBeacon alertBeacon; @Autowired private GridFsRepository gridFsRepository; @Autowired private ImportSetRepository importSetRepository; @Autowired private ItemRepository itemRepository; @Autowired private ItemMetadataService itemMetadataService; @Autowired private FileTransferService sftpFileTransferService; @Autowired private ZipInputFileExtractorService apipZipInputFileExtractorService; @Value(value = "${tib.file.pathname:/wurst${tib.file.importfolder:/tib-imports}}") // do NOT default to blank file path, unless you hate this project private String tempFileDirectory; @Override public ImportSet saveImportSet(final ImportSet importSet) { final ImportSet newImportSet = this.importSetRepository.save(importSet); return newImportSet; } @Override public ImportSet getImportSet(final String importSetId) { return this.importSetRepository.findOne(importSetId); } @Async @Override public void importFileSet(final ImportSet importSet) { for (final ImportFile importFile : importSet.getImportFiles()) { try { LOGGER.debug("importing file:" + importFile.getPathName()); File fileToImport = null; switch (importSet.getImportType()) { case SFTP: fileToImport = this.sftpFileTransferService.getFile(importSet.getId(), tibImportDirectory + "/" + SFTP_TENANT_FOLDER_PREFIX + importSet.getTenantId() + "/" + importFile.getPathName()); break; case STAGED_FILE: fileToImport = new File(importFile.getPathName()); break; case MULTIPART_REQUEST: fileToImport = importFile.getFile(); break; default: break; } importFile.setFile(fileToImport); processImportFile(importFile, importSet.getId(), importSet.getTenantId(), importSet.getItemBank()); importFile.setImportCompleteTime(new DateTime()); } catch (final TestItemBankException tibE) { importFile.setImportStatus(ImportStatus.FAILED); importFile.addMessage("unknown", tibE.getMessageCode(), tibE.getMessageArgs()); LOGGER.error("unexpected error importing item:", tibE); } catch (final Exception e) { importFile.setImportStatus(ImportStatus.FAILED); importFile.addMessage("unexpected.error"); LOGGER.error("unexpected error importing item:", e); } finally { FileUtils.deleteQuietly(new File(this.tempFileDirectory)); } saveImportSet(importSet); } importSet.setImportStatus(ImportStatus.IMPORT_COMPLETE); importSet.setImportCompleteTime(new DateTime()); this.alertBeacon.sendAlert(MnaSeverity.INFO, MnaAlertType.TIB_IMPORT.name(), "Import set " + importSet.getId() + " done importing"); saveImportSet(importSet); } /** * processes import file. * * @param importFile to process. * @param importSetId import set it is a part of. */ private void processImportFile(final ImportFile importFile, final String importSetId, final String tenantId, final String itemBank) { int itemsImported = 0; final File fileToImport = importFile.getFile(); if (fileToImport == null || !fileToImport.exists()) { importFile.addMessage("file.not.found"); importFile.setImportStatus(ImportStatus.FILE_NOT_FOUND); } else { try { // DO WE NEED THIS METADATA? final BasicDBObject metadata = new BasicDBObject(); metadata.put("fileName", fileToImport.getName()); metadata.put("importSetId", importSetId); final GridFSFile gridfsFile = this.gridFsRepository.save(fileToImport, fileToImport.getName(), metadata); final String gridfsId = gridfsFile.getId().toString(); // extract items in file final List<ApipItemContent> itemContents = this.apipZipInputFileExtractorService .createItems(importFile.getFile(), gridfsId, tenantId, itemBank); // save/validate items for (final ApipItemContent itemContent : itemContents) { final ItemValidator itemValidator = new ItemValidator(); final Item item = itemContent.getItem(); final String itemIdentifier = item.getIdentifier(); try { final Errors errors = validate(itemValidator, itemContent); if (errors.hasErrors()) { importFile.setImportStatus(ImportStatus.FAILED); for (final ObjectError objectError : errors.getAllErrors()) { importFile.addMessage(item.getIdentifier(), objectError.getCode(), (String[]) objectError.getArguments()); } } else { String itemfsId = ""; if (itemContents.size() == 1) { itemfsId = gridfsId; } else { final BasicDBObject itemGridFSMetadata = new BasicDBObject(); final String itemFileName = item.getIdentifier() + "_" + fileToImport.getName(); itemGridFSMetadata.put("fileName", itemFileName); itemGridFSMetadata.put("importSetId", importSetId); itemGridFSMetadata.put("itemIdentifier", item.getIdentifier()); itemGridFSMetadata.put("version", item.getVersion()); itemGridFSMetadata.put("originalFileId", gridfsId); final GridFSFile itemfsFile = this.gridFsRepository.save(itemContent.getItemZip(), itemFileName, itemGridFSMetadata); itemfsId = itemfsFile.getId().toString(); } item.setItemZipGridId(itemfsId); this.itemRepository.addItem(item); itemsImported++; importFile.addMessage(itemIdentifier, "item.successfully.added", new String[] { item.getIdentifier(), item.getVersion() }); for (final Entry<String, Object> metadataEntry : item.getAllIncludedMetatdata() .entrySet()) { this.itemMetadataService.saveNewItemMetadata(tenantId, metadataEntry.getKey(), metadataEntry.getValue()); } } } catch (final TestItemBankException tibE) { importFile.setImportStatus(ImportStatus.FAILED); importFile.addMessage(itemIdentifier, tibE.getMessageCode(), tibE.getMessageArgs()); } catch (final DuplicateKeyException e) { importFile.setImportStatus(ImportStatus.FAILED); importFile.addMessage(itemIdentifier, "item.already.exists", new String[] { item.getIdentifier(), item.getVersion() }); } } if (importFile.getImportStatus() != ImportStatus.FAILED) { importFile.addMessage("Successful import"); importFile.setImportStatus(ImportStatus.IMPORT_COMPLETE); } final String message = itemsImported + " items created from import file " + importFile.getPathName() + " from import set:" + importSetId; this.alertBeacon.sendAlert(MnaSeverity.INFO, MnaAlertType.TIB_IMPORT.name(), message); } catch (final TestItemBankException tibE) { importFile.setImportStatus(ImportStatus.FAILED); importFile.addMessage("", tibE.getMessageCode(), tibE.getMessageArgs()); sendZipFileMonitoringAndAlertingErrors(tibE.getMessageCode(), new String[] { importFile.getPathName() }); } catch (final Exception e) { LOGGER.error("unknown error importing file", e); importFile.addMessage("unexpected.error"); importFile.setImportStatus(ImportStatus.FAILED); } } } private Errors validate(final ItemValidator inItemValidator, final ApipItemContent inItemContent) { final Errors errors = new BindException(inItemContent.getItem(), "item"); if (inItemContent.getErrors() == null || inItemContent.getErrors().hasErrors()) { for (final ObjectError error : inItemContent.getErrors().getAllErrors()) { sendZipFileMonitoringAndAlertingErrors(error.getCode(), error.getArguments()); } errors.addAllErrors(inItemContent.getErrors()); } else { inItemValidator.validate(inItemContent.getItem(), errors); } return errors; } /** * this is not a great way to do to this. * For now we are adding these log entries in place of monitoring and alerting hooks. * Once monitoring and alerting is ready can come up with a better way to * map user friendly messages. * * @param errorCode . * @param args . */ private void sendZipFileMonitoringAndAlertingErrors(final String errorCode, final Object[] args) { String firstArg = ""; if (args != null && args[0] != null) { firstArg = args[0].toString(); } if ("item.invalid.zip.missingMetadata".equals(errorCode)) { this.alertBeacon.sendAlert(MnaSeverity.ERROR, MnaAlertType.TIB_IMPORT.name(), "missing metadata file is missing for item:"); } else if ("item.invalid.zip.missingResource".equals(errorCode)) { this.alertBeacon.sendAlert(MnaSeverity.ERROR, MnaAlertType.TIB_IMPORT.name(), "missing resource in zip file for item:" + firstArg); } else if ("apip.zip.extractor.failure.mainfest".equals(errorCode)) { this.alertBeacon.sendAlert(MnaSeverity.ERROR, MnaAlertType.TIB_IMPORT.name(), "missing imsmainifest file in :" + firstArg); } } @Scheduled(cron = "${tib.clean.files.cron.trigger:}") @Override public void cleanSftpDirectory() { this.sftpFileTransferService.cleanDirectory(tibImportDirectory, maxImportFileAge); } @Override public void setSftpFileTransferService(final FileTransferService inSftpFileTransferService) { this.sftpFileTransferService = inSftpFileTransferService; } }