Java tutorial
/******************************************************************************* * Educational Online Test Delivery System * Copyright (c) 2013 American Institutes for Research * * Distributed under the AIR Open Source License, Version 1.0 * See accompanying file AIR-License-1_0.txt or at * http://www.smarterapp.org/documents/American_Institutes_for_Research_Open_Source_Software_License.pdf ******************************************************************************/ package org.opentestsystem.authoring.testspecbank.service.impl; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import com.mongodb.DBObject; import com.mongodb.gridfs.GridFSDBFile; import com.mongodb.gridfs.GridFSFile; import org.apache.commons.lang.builder.ToStringBuilder; import org.joda.time.DateTime; import org.opentestsystem.authoring.testspecbank.client.tib.TestItemBankClientInterface; import org.opentestsystem.authoring.testspecbank.client.tib.ValidationErrorCode; import org.opentestsystem.authoring.testspecbank.domain.ExportPackage; import org.opentestsystem.authoring.testspecbank.domain.ExportPackageStatus; import org.opentestsystem.authoring.testspecbank.domain.MnaAlertType; import org.opentestsystem.authoring.testspecbank.domain.Permissions; import org.opentestsystem.authoring.testspecbank.domain.Purpose; import org.opentestsystem.authoring.testspecbank.domain.TestSpecification; import org.opentestsystem.authoring.testspecbank.domain.TibExportDetails; import org.opentestsystem.authoring.testspecbank.domain.search.TestSpecificationSearchRequest; import org.opentestsystem.authoring.testspecbank.domain.tib.ExportItemClientObj; import org.opentestsystem.authoring.testspecbank.domain.tib.ExportSetClientObj; import org.opentestsystem.authoring.testspecbank.persistence.GridFsRepository; import org.opentestsystem.authoring.testspecbank.persistence.TestSpecificationRepository; import org.opentestsystem.authoring.testspecbank.service.FileManagerService; import org.opentestsystem.authoring.testspecbank.service.FileTransferService; import org.opentestsystem.authoring.testspecbank.service.PublisherSingletons; import org.opentestsystem.authoring.testspecbank.service.TestSpecificationService; import org.opentestsystem.shared.exception.LocalizedException; import org.opentestsystem.shared.exception.RestException; import org.opentestsystem.shared.mna.client.domain.MnaSeverity; import org.opentestsystem.shared.mna.client.service.AlertBeacon; import org.opentestsystem.shared.search.domain.SearchResponse; import org.opentestsystem.shared.security.domain.SbacUser; import org.opentestsystem.shared.security.service.UserService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.dao.DuplicateKeyException; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.springframework.util.Assert; import org.springframework.util.StringUtils; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.SAXException; import javax.xml.transform.TransformerException; import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.zip.DataFormatException; import java.util.zip.Inflater; import tds.common.ValidationError; @Service("testSpecificationService") public class TestSpecificationServiceImpl implements TestSpecificationService { private static final Logger LOGGER = LoggerFactory.getLogger(TestSpecificationServiceImpl.class); public static final int BUFFER_SIZE = 4096; @Autowired private transient TestSpecificationRepository testSpecificationRepository; @Autowired private transient GridFsRepository gridFsRepository; @Autowired private transient UserService userService; @Autowired private AlertBeacon alertBeacon; @Autowired private FileManagerService fileManagerService; @Autowired private FileTransferService testPackagerFileTransferService; @Autowired private FileTransferService testItemBankFileTransferService; @Autowired private TestItemBankClientInterface testItemBankClient; @Value("${tsb.dtd.validation}") private String testSpecDtdValidation; @Value("${tsb.dtd.url}") private String testSpecDtdUrl; @Override public TestSpecification saveTestSpecification(final TestSpecification testSpecification) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Saving TestSpecification; TestSpecifcation: " + testSpecification.toString()); } TestSpecification savedTestSpecification = null; try { if (Boolean.valueOf(this.testSpecDtdValidation)) { final byte[] decompressedXml = decompress(testSpecification.getSpecificationXml()); validateXmlWithDtd(decompressedXml); } Assert.isNull(testSpecification.getLastUpdatedDate()); testSpecification.setLastUpdatedDate(new DateTime()); testSpecification.setTenantSet(Sets.newHashSet(testSpecification.getTenantId())); if (testSpecification.getSpecificationXml() != null) { if (Boolean.valueOf(this.testSpecDtdValidation)) { final byte[] decompressedXml = decompress(testSpecification.getSpecificationXml()); validateXmlWithDtd(decompressedXml); final DBObject metadata = new GridFSDBFile(); metadata.put("contentType", "application/xml"); final GridFSFile gridFsFile = this.gridFsRepository.save( testSpecification.getSpecificationXml(), generateGridFsFilename(testSpecification), metadata); testSpecification.setSpecificationXmlGridFsId(gridFsFile.getId().toString()); } } savedTestSpecification = this.testSpecificationRepository.save(testSpecification); final String message = "Test Specification successfully stored; name: " + testSpecification.getName() + ", version: " + testSpecification.getVersion(); this.alertBeacon.sendAlert(MnaSeverity.INFO, MnaAlertType.TEST_SPEC_SAVED.name(), message); } catch (final Exception e) { final Map<String, String[]> parameterMap = ImmutableMap.of("name", new String[] { testSpecification.getName() }, "version", new String[] { testSpecification.getVersion() }, "tenantId", new String[] { testSpecification.getTenantId() }); final TestSpecificationSearchRequest searchReq = new TestSpecificationSearchRequest(parameterMap); final SearchResponse<TestSpecification> response = this.testSpecificationRepository.search(searchReq); boolean fallDownGoBoom = true; if (response.getTotalCount() > 0) { boolean wipeTheSlateClean = true; if (e instanceof DuplicateKeyException) { LOGGER.warn("specs already saved for tenantId:" + testSpecification.getTenantId() + ", name: " + testSpecification.getName() + ", version: " + testSpecification.getVersion()); for (final TestSpecification existingTestSpec : response.getSearchResults()) { if (existingTestSpec.getPurpose() == testSpecification.getPurpose()) { savedTestSpecification = existingTestSpec; fallDownGoBoom = false; wipeTheSlateClean = false; } } } if (wipeTheSlateClean) { for (final TestSpecification existingTestSpec : response.getSearchResults()) { this.gridFsRepository.delete(existingTestSpec.getSpecificationXmlGridFsId()); } this.testSpecificationRepository.delete(response.getSearchResults()); LOGGER.error("specs deleted for tenantId:" + testSpecification.getTenantId() + ", name: " + testSpecification.getName() + ", version: " + testSpecification.getVersion()); } } if (fallDownGoBoom) { throw e; } } return savedTestSpecification; } @Override public TestSpecification updateTenantSet(final String testSpecificationId, final Set<String> tenantSet) { final TestSpecification testSpecification = getTestSpecification(testSpecificationId, false); if (testSpecification == null) { throw new LocalizedException("testspec.id.invalid"); } if (testSpecification.isRetired()) { throw new LocalizedException("testspec.retired.update"); } if (testSpecification.getExportPackage() != null && (testSpecification.getExportPackage().getStatus() == ExportPackageStatus.SUBMITTED || testSpecification.getExportPackage() .getStatus() == ExportPackageStatus.PENDING_ITEM_EXPORT || testSpecification.getExportPackage() .getStatus() == ExportPackageStatus.PENDING_PACKAGE_CREATION || testSpecification.getExportPackage().getStatus() == ExportPackageStatus.PENDING_SFTP)) { throw new LocalizedException("testspec.submitted.update"); } if (tenantSet == null || tenantSet.size() < 1) { throw new LocalizedException("testspec.tenant.size"); } testSpecification.setTenantSet(tenantSet); testSpecification.setLastUpdatedDate(new DateTime()); return this.testSpecificationRepository.save(testSpecification); } @Override public TestSpecification getTestSpecification(final String testSpecificationId, final boolean excludeXml) { final TestSpecification testSpecification = this.testSpecificationRepository.findOne(testSpecificationId); if (!excludeXml) { populateTestSpecification(testSpecification); } return testSpecification; } @Override public SearchResponse<TestSpecification> searchTestSpecifications(final Map<String, String[]> parameterMap, final boolean includeXml) { if (LOGGER.isDebugEnabled()) { LOGGER.debug( "Searching TestSpecification, params: " + ToStringBuilder.reflectionToString(parameterMap)); } final TestSpecificationSearchRequest searchRequest = new TestSpecificationSearchRequest(parameterMap); if (searchRequest.isValid()) { final SearchResponse<TestSpecification> searchResponse = this.testSpecificationRepository .search(searchRequest); if (includeXml) { for (final TestSpecification testSpecification : searchResponse.getSearchResults()) { populateTestSpecification(testSpecification); } } return searchResponse; } throw new RestException("testspec.search.invalidSearchCriteria"); } @Override public TestSpecification retireTestSpecification(final String testSpecificationId, final boolean undoRetirement) { final TestSpecification testSpecification = getTestSpecification(testSpecificationId, false); testSpecification.setRetired(!undoRetirement); if (testSpecification.getExportPackage() != null && (testSpecification.getExportPackage().getStatus() == ExportPackageStatus.SUBMITTED || testSpecification.getExportPackage() .getStatus() == ExportPackageStatus.PENDING_ITEM_EXPORT || testSpecification.getExportPackage() .getStatus() == ExportPackageStatus.PENDING_PACKAGE_CREATION || testSpecification.getExportPackage().getStatus() == ExportPackageStatus.PENDING_SFTP)) { throw new LocalizedException("testspec.export.retire"); } if (undoRetirement) { this.alertBeacon.sendAlert(MnaSeverity.INFO, MnaAlertType.TEST_SPEC_RESTORED.name(), "Test Specification restored: " + testSpecification.getId()); } else { this.alertBeacon.sendAlert(MnaSeverity.INFO, MnaAlertType.TEST_SPEC_RETIRED.name(), "Test Specification retired: " + testSpecification.getId()); } return this.testSpecificationRepository.save(testSpecification); } @Override public boolean isAdminUser() { final SbacUser user = this.userService.getCurrentUser(); return user.hasPermission(Permissions.TEST_SPEC_ADMIN.toSpringRoleName()); } @Override public List<TestSpecification> getTestSpecificationsByExportPackageStatusIn( final Set<ExportPackageStatus> statuses, final boolean includeXml) { final List<TestSpecification> foundSpecs = this.testSpecificationRepository .findByExportPackageStatusIn(statuses); if (includeXml) { for (final TestSpecification spec : foundSpecs) { populateTestSpecification(spec); } } return foundSpecs; } @Override public TestSpecification requestExportPackage(final String testSpecificationId) { final TestSpecification testSpecification = getTestSpecification(testSpecificationId, false); if (testSpecification.isRetired()) { throw new LocalizedException("testspec.retired.export"); } testSpecification.setExportPackage(new ExportPackage()); testSpecification.getExportPackage().setStatus(ExportPackageStatus.SUBMITTED); testSpecification.getExportPackage().setTimeRequested(new DateTime()); return this.testSpecificationRepository.save(testSpecification); } @Override public TestSpecification retryExportPackage(final String testSpecificationId) { final TestSpecification testSpecification = getTestSpecification(testSpecificationId, false); if (testSpecification == null || testSpecification.getExportPackage() == null) { throw new LocalizedException("exportPackage.not.exists"); } else if (testSpecification.isRetired()) { throw new LocalizedException("testspec.retired.export"); } testSpecification.getExportPackage().setStatus(ExportPackageStatus.SUBMITTED); testSpecification.getExportPackage().setTimeRequested(new DateTime()); testSpecification.getExportPackage().setStatusMessage(null); testSpecification.getExportPackage().setTibExportDetails(null); testSpecification.getExportPackage().setZipFileNames(null); testSpecification.getExportPackage().setExportCompleted(null); return this.testSpecificationRepository.save(testSpecification); } @Async @Override public void loadTestSpecification(final TestSpecification testSpecification) { try { testSpecification.getExportPackage().setStatus(ExportPackageStatus.PENDING_PACKAGE_CREATION); if (Purpose.ADMINISTRATION.equals(testSpecification.getPurpose()) || Purpose.COMPLETE.equals(testSpecification.getPurpose())) { // create export set request from items in spec xml final ExportSetClientObj exportSetRequest = new ExportSetClientObj(); exportSetRequest.setTenantId(testSpecification.getTenantId()); exportSetRequest.setItems(parseTibItemsFromSpecXml(testSpecification.getPurpose(), testSpecification.getSpecificationXml())); // request export from TIB and store details in package object if (!exportSetRequest.getItems().isEmpty()) { final ExportSetClientObj exportSet = wrapTibClientExportSetCall(true, exportSetRequest, null); testSpecification.getExportPackage().setTibExportDetails(new TibExportDetails(exportSet)); testSpecification.getExportPackage().setStatus(ExportPackageStatus.PENDING_ITEM_EXPORT); } } this.testSpecificationRepository.save(testSpecification); } catch (final RuntimeException e) { handleFailedExport(testSpecification.getId(), e); } } @Async @Override public void checkTibExportStatus(final TestSpecification testSpecification) { try { final ExportSetClientObj exportSet = wrapTibClientExportSetCall(false, null, testSpecification.getExportPackage().getTibExportDetails().getId()); testSpecification.getExportPackage().setTibExportDetails(new TibExportDetails(exportSet)); switch (testSpecification.getExportPackage().getTibExportDetails().getStatus()) { case FAILED: testSpecification.getExportPackage().setStatus(ExportPackageStatus.FAILED); testSpecification.getExportPackage().setStatusMessage("tib.export.failed"); this.alertBeacon.sendAlert(MnaSeverity.ERROR, MnaAlertType.EXPORT_PACKAGE_FAILED.name(), "Export Package Failed: " + testSpecification.getId() + " - " + testSpecification.getExportPackage().getStatusMessage()); break; case EXPORT_COMPLETE: testSpecification.getExportPackage().setStatus(ExportPackageStatus.PENDING_PACKAGE_CREATION); break; default: break; } this.testSpecificationRepository.save(testSpecification); } catch (final RuntimeException e) { handleFailedExport(testSpecification.getId(), e); } } @Async @Override public void buildExportPackageZip(final TestSpecification testSpecification) { try { final String downloadDirectoryName = generatePackageZipFilename(testSpecification, ""); final File downloadDirectory = this.fileManagerService.initializeCleanDirectory(downloadDirectoryName); // download item packages if (testSpecification.getExportPackage().getTibExportDetails() != null && !StringUtils.isEmpty(testSpecification.getExportPackage().getTibExportDetails().getId())) { testSpecification.getExportPackage().setStatus(ExportPackageStatus.DOWNLOADING_ITEMS); this.testSpecificationRepository.save(testSpecification); final ExportSetClientObj exportSet = wrapTibClientExportSetCall(false, null, testSpecification.getExportPackage().getTibExportDetails().getId()); this.testItemBankFileTransferService.downloadFile(exportSet.getZipFileName(), downloadDirectoryName); testSpecification.getExportPackage().getTibExportDetails().setDownloadComplete(true); } // write test spec xml this.fileManagerService.writeFile(downloadDirectoryName + "/" + "test_specification.xml", testSpecification.getSpecificationXml()); // build zip file this.fileManagerService.buildZipFromDirectory(downloadDirectory, generatePackageZipFilename(testSpecification, ".zip")); testSpecification.getExportPackage().setStatus(ExportPackageStatus.PENDING_SFTP); this.testSpecificationRepository.save(testSpecification); } catch (final RuntimeException e) { handleFailedExport(testSpecification.getId(), e); } } @Override public Optional<ValidationError> deleteTestSpecification(final String testSpecificationName) { TestSpecification testSpecification = testSpecificationRepository.findOneByName(testSpecificationName); if (testSpecification == null) { return Optional.of(new ValidationError(ValidationErrorCode.TEST_SPECIFICATION_NOT_FOUND, String.format("Could not find test specification for key '%s'", testSpecificationName))); } testSpecificationRepository.delete(testSpecification); if (testSpecification.getSpecificationXmlGridFsId() != null) { gridFsRepository.delete(testSpecification.getSpecificationXmlGridFsId()); } return Optional.empty(); } /** * wrap TIB Client interactions to capture errors and show a better message onscreen */ private ExportSetClientObj wrapTibClientExportSetCall(final boolean requestExport, final ExportSetClientObj exportSetRequest, final String exportId) { ExportSetClientObj exportSet = null; try { exportSet = requestExport ? this.testItemBankClient.requestExport(exportSetRequest) : this.testItemBankClient.getExportSet(exportId); } catch (final Exception e) { throw new LocalizedException("testspec.export.tib.communication.error", e); } return exportSet; } @Async @Override public void transferExportPackage(final TestSpecification testSpecification) { try { // determine local file name for the export (file should already exist) final String fileName = generatePackageZipFilename(testSpecification, ".zip"); // write file to SFTP site (for each tenant in tenantSet) testSpecification.getExportPackage().setZipFileNames(new ArrayList<String>()); for (final String tenantId : testSpecification.getTenantSet()) { this.testPackagerFileTransferService.writeFile(fileName, "tenant_" + tenantId, fileName); testSpecification.getExportPackage().getZipFileNames().add("tenant_" + tenantId + "/" + fileName); } testSpecification.getExportPackage().setStatus(ExportPackageStatus.COMPLETE); testSpecification.getExportPackage().setExportCompleted(new DateTime()); this.alertBeacon.sendAlert(MnaSeverity.INFO, MnaAlertType.EXPORT_PACKAGE_COMPLETE.name(), "Export package complete: " + testSpecification.getId()); this.testSpecificationRepository.save(testSpecification); } catch (final RuntimeException e) { handleFailedExport(testSpecification.getId(), e); } } private void handleFailedExport(final String testSpecificationId, final RuntimeException exception) { LOGGER.error("Error during package export", exception); final TestSpecification testSpecification = getTestSpecification(testSpecificationId, false); if (testSpecification == null) { throw new LocalizedException("testspec.id.invalid"); } if (testSpecification.getExportPackage() == null) { throw new LocalizedException("testspec.export.required"); } testSpecification.getExportPackage().setStatus(ExportPackageStatus.FAILED); testSpecification.setLastUpdatedDate(new DateTime()); testSpecification.getExportPackage().setStatusMessage( exception instanceof LocalizedException ? exception.getCause().getMessage() : "unexpected.error"); this.alertBeacon.sendAlert(MnaSeverity.ERROR, MnaAlertType.EXPORT_PACKAGE_FAILED.name(), "Export Package Failed: " + testSpecification.getId() + " - " + testSpecification.getExportPackage().getStatusMessage()); this.testSpecificationRepository.save(testSpecification); if (!(exception instanceof LocalizedException)) { throw exception; } } private final String generateGridFsFilename(final TestSpecification testSpecification) { final StringBuilder sb = new StringBuilder(); sb.append(testSpecification.getName()); sb.append("_"); sb.append(testSpecification.getPurpose()); sb.append("_"); sb.append(testSpecification.getVersion()); sb.append("_"); sb.append(testSpecification.getLastUpdatedDate()); sb.append("_"); sb.append(testSpecification.getTenantId()); sb.append(".xml"); return sb.toString(); } private String generatePackageZipFilename(final TestSpecification testSpecification, final String desiredExtension) { final StringBuilder fileName = new StringBuilder(); fileName.append("test_package_"); fileName.append(testSpecification.getName()); fileName.append("_"); fileName.append(testSpecification.getPurpose()); fileName.append("_v"); fileName.append(testSpecification.getVersion()); if (!StringUtils.isEmpty(desiredExtension)) { fileName.append(desiredExtension); } return fileName.toString(); } private final byte[] decompress(final byte[] testSpecificationXml) { final Inflater inflater = new Inflater(); inflater.setInput(testSpecificationXml); final ByteArrayOutputStream baos = new ByteArrayOutputStream(testSpecificationXml.length); byte[] outBytes = null; try { final byte[] buffer = new byte[BUFFER_SIZE]; while (!inflater.finished()) { final int count = inflater.inflate(buffer); baos.write(buffer, 0, count); } baos.close(); outBytes = baos.toByteArray(); } catch (final IOException | DataFormatException e) { throw new LocalizedException("testspec.xml.compress.error", e); } return outBytes; } private void validateXmlWithDtd(final byte[] testSpecificationXml) { try { // since the XML is created by Test Authoring and doesn't contain a DOCTYPE, it is added here for validation final ByteArrayOutputStream baos = new ByteArrayOutputStream(); PublisherSingletons.getInstance(this.testSpecDtdUrl).getDOCUMENT_TRANSFORMER().transform( new StreamSource(new ByteArrayInputStream(testSpecificationXml)), new StreamResult(baos)); // re-parse the transformed file, this time with DTD, to check validity PublisherSingletons.getInstance(this.testSpecDtdUrl).getDOCUMENT_BUILDER() .parse(new ByteArrayInputStream(baos.toByteArray())); } catch (SAXException | TransformerException | IOException e) { throw new LocalizedException("testspec.spec.xml.invalid", new String[] { e.getMessage() }, e); } } private void populateTestSpecification(final TestSpecification testSpecification) { if (testSpecification != null && testSpecification.getSpecificationXmlGridFsId() != null) { final ByteArrayOutputStream ret = new ByteArrayOutputStream(); try { final GridFSDBFile grid = this.gridFsRepository .getById(testSpecification.getSpecificationXmlGridFsId()); grid.writeTo(ret); ret.flush(); } catch (final IOException e) { throw new LocalizedException("testspec.spec.xml.notfound", new String[] { testSpecification.getSpecificationXmlGridFsId() }, e); } if (ret.size() > 0) { final byte[] decompressedXml = decompress(ret.toByteArray()); testSpecification.setSpecificationXml(decompressedXml); } else { LOGGER.error("Test specification xml contains zero bytes: " + testSpecification.getName()); throw new LocalizedException("testspec.spec.xml.notfound"); } } } private List<ExportItemClientObj> parseTibItemsFromSpecXml(final Purpose purpose, final byte[] specXml) { final String identifierExpression = "/testspecification/" + purpose.name().toLowerCase() + "/itempool/testitem/identifier"; final NodeList identifierList = TestPackagerXmlParser.parseNodeList(identifierExpression, specXml); final List<ExportItemClientObj> exportItems = new ArrayList<ExportItemClientObj>(); for (int i = 0; i < identifierList.getLength(); i++) { final Node identifier = identifierList.item(i); final ExportItemClientObj exportItem = new ExportItemClientObj(); exportItem.setIdentifier(identifier.getAttributes().getNamedItem("label").getNodeValue()); exportItem.setVersion(identifier.getAttributes().getNamedItem("version").getNodeValue()); exportItems.add(exportItem); } // now let's pick up passages final String identifierPassageExpression = "/testspecification/" + purpose.name().toLowerCase() + "/itempool/passage/identifier"; final NodeList identifierPassageList = TestPackagerXmlParser.parseNodeList(identifierPassageExpression, specXml); for (int i = 0; i < identifierPassageList.getLength(); i++) { final Node identifier = identifierPassageList.item(i); final ExportItemClientObj exportItem = new ExportItemClientObj(); exportItem.setIdentifier(identifier.getAttributes().getNamedItem("label").getNodeValue()); exportItem.setVersion(identifier.getAttributes().getNamedItem("version").getNodeValue()); exportItems.add(exportItem); } return exportItems; } }