Java tutorial
/******************************************************************************* * Educational Online Test Delivery System * Copyright (c) 2013 American Institutes for Research * * Distributed under the AIR Open Source License, Version 1.0 * See accompanying file AIR-License-1_0.txt or at * http://www.smarterapp.org/documents/American_Institutes_for_Research_Open_Source_Software_License.pdf ******************************************************************************/ package org.opentestsystem.delivery.testreg.rest; import static org.opentestsystem.delivery.testreg.rest.FileType.TXT; import static org.opentestsystem.delivery.testreg.rest.FileType.XLSX; import static org.opentestsystem.delivery.testreg.upload.ExcelUtils.getRowNum; import static org.opentestsystem.delivery.testreg.upload.parser.ParserTextUtils.areAllElementsNull; import static org.opentestsystem.delivery.testreg.upload.parser.ParserTextUtils.isEmptyRecord; import static org.opentestsystem.delivery.testreg.upload.parser.ParserTextUtils.padEmptyIfNoColumnAtEnd; import static org.opentestsystem.delivery.testreg.upload.parser.ParserTextUtils.trimRecords; import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import javax.annotation.Resource; import javax.servlet.http.HttpServletResponse; import org.apache.commons.io.FilenameUtils; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; import org.opentestsystem.delivery.testreg.domain.FileUploadResponse; import org.opentestsystem.delivery.testreg.domain.FileUploadSummary; import org.opentestsystem.delivery.testreg.domain.FormatType; import org.opentestsystem.delivery.testreg.domain.TestRegistrationBase; import org.opentestsystem.delivery.testreg.persistence.criteria.dependencyresolvers.Sb11DependencyResolverInvoker; import org.opentestsystem.delivery.testreg.rest.ValidationMessage.ValidationMessageType; import org.opentestsystem.delivery.testreg.service.FileUploadService; import org.opentestsystem.delivery.testreg.service.TestRegPersister; import org.opentestsystem.delivery.testreg.upload.DataRecord; import org.opentestsystem.delivery.testreg.upload.ExcelUtils; import org.opentestsystem.delivery.testreg.upload.ExcelUtils.ExcelRowMapper; import org.opentestsystem.delivery.testreg.upload.ExcelUtils.ExcelWorksheetProcessor; import org.opentestsystem.delivery.testreg.upload.FileFormatTypeAppender; import org.opentestsystem.delivery.testreg.upload.FileUploadUtils; import org.opentestsystem.delivery.testreg.upload.RowMetadata; import org.opentestsystem.delivery.testreg.upload.TextFileUtils; import org.opentestsystem.delivery.testreg.upload.TextFileUtils.LineMapper; import org.opentestsystem.delivery.testreg.upload.parser.ParserResult; import org.opentestsystem.delivery.testreg.upload.parser.UploadFileParser; import org.opentestsystem.shared.exception.LocalizedException; import org.opentestsystem.shared.exception.RestException; import org.opentestsystem.shared.mna.client.domain.MnaSeverity; import org.opentestsystem.shared.mna.client.service.AlertBeacon; import org.opentestsystem.shared.mna.client.service.MetricClient; import org.opentestsystem.shared.web.AbstractRestController; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.security.access.annotation.Secured; import org.springframework.stereotype.Controller; import org.springframework.util.CollectionUtils; import org.springframework.validation.BindingResult; import org.springframework.validation.FieldError; import org.springframework.validation.ValidationUtils; import org.springframework.validation.Validator; import org.springframework.web.bind.annotation.ModelAttribute; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.multipart.MultipartFile; import com.google.common.collect.Lists; import com.mongodb.gridfs.GridFSDBFile; @Controller public class FileUploadDataController extends AbstractRestController { private static final int DEFAULT_ERROR_THRESHOLD = 500; private static final int HEADER_ROW = 1; @Resource(name = "fileParserMap") private Map<FileType, UploadFileParser<Map<String, List<DataRecord>>>> fileParserMap; @Autowired @Qualifier("fileUploadValidator") private Validator fileUploadValidator; @Autowired ExcelUtils excelUtils; private final TextFileUtils textFileUtils = new TextFileUtils(); @Autowired private FileUploadService fileUploadService; @Autowired private TestRegPersister entityService; @Autowired private FileUploadUtils fileUploadUtils; @Autowired private Sb11DependencyResolverInvoker sb11DependencyResolverInvoker; @Autowired private FileFormatTypeAppender fileFormatTypeAppender; @Autowired private MetricClient metricClient; @Autowired private AlertBeacon alertBeacon; @Value("${testreg.fileupload.errorcount.threshold:50}") private String errorCountThreshold; /** * Upload File. * * @param testRegFile * to be saved. * @returns response FileUploadResponse. */ @ResponseStatus(HttpStatus.CREATED) @RequestMapping(value = "/uploadFile", method = RequestMethod.POST, produces = { MediaType.APPLICATION_JSON_VALUE }) @Secured({ "ROLE_Accommodations Upload", "ROLE_Student Upload", "ROLE_Entity Upload", "ROLE_StudentGroup Upload", "ROLE_User Upload", "ROLE_ExplicitEligibility Upload" }) @ResponseBody public FileUploadResponse uploadFile(@RequestParam("uploadFile") final MultipartFile testRegFile, @RequestParam final String formatType) throws Exception { FileUploadResponse response; final long start = System.currentTimeMillis(); if (!FileType.isValidType(testRegFile.getOriginalFilename())) { throw new LocalizedException("file.invalid.fileextension", new String[] { FilenameUtils.getExtension(testRegFile.getOriginalFilename()), Arrays.toString(FileType.values()) }); } else { /* * Following line inserts FormatType of the file as first row in the file. */ // InputStream inputStream = fileFormatTypeAppender.forFile(testRegFile).insert(formatType); final long middle = System.currentTimeMillis(); this.metricClient.sendPerformanceMetricToMna("upload begin (ms)", middle - start); response = this.fileUploadService.saveFile(testRegFile.getOriginalFilename(), testRegFile.getInputStream(), formatType); this.metricClient.sendPerformanceMetricToMna( buildMetricMessage(response.getFileGridFsId(), "upload end"), System.currentTimeMillis() - middle); this.alertBeacon.sendAlert(MnaSeverity.INFO, "ART_" + FormatType.valueOf(formatType).getFormatName() + "_UPLOAD", response.getMessage()); } return response; } /** * Validates a file given its format and gridFsId. * * @param gridFsId * A {@link ModelAttribute} whose value is bound from the request mapping variables. * @param result * An interface for binding results of all forms of validation * @param response * HttpServletResponse for sending HTTP-specific responses * @return Returns {@link FileValidationResult} * @throws Exception */ @RequestMapping(value = "/validateFile/{gridFsId}", method = RequestMethod.GET, produces = { MediaType.APPLICATION_JSON_VALUE }) @Secured({ "ROLE_Accommodations Upload", "ROLE_Student Upload", "ROLE_Entity Upload", "ROLE_StudentGroup Upload", "ROLE_User Upload", "ROLE_ExplicitEligibility Upload" }) @ResponseBody public List<FileValidationResult> validate(@ModelAttribute("gridFsId") final String gridFsId, final BindingResult result, final HttpServletResponse response) throws Exception { final long start = System.currentTimeMillis(); final GridFSDBFile file = getGridFSDBFile(gridFsId); this.metricClient.sendPerformanceMetricToMna(buildMetricMessage(gridFsId, "validateFile->getGridFSDBFile"), System.currentTimeMillis() - start); long startMarker = System.currentTimeMillis(); final List<FileValidationResult> returnList = Lists.newArrayList(); // fileType cannot be null final FileType fileType = FileType.findByFilename(file.getFilename()); // throws IllegalArgumentException when no enum is found from file-extension final UploadFileParser<Map<String, List<DataRecord>>> fileParser = this.fileParserMap.get(fileType); ParserResult<Map<String, List<DataRecord>>> parsedResult = null; try { parsedResult = fileParser.parse(file.getInputStream(), retrieveFormatTypeFromFileMetadata(file)); } catch (final LocalizedException loc) { final FileValidationResult validationResult = new FileValidationResult(); validationResult.addError(new ValidationMessage("Invalid File type: " + loc.getLocalizedMessage(), ValidationMessageType.FATAL_ERROR)); returnList.add(validationResult); return returnList; } this.metricClient.sendPerformanceMetricToMna(buildMetricMessage(gridFsId, "validateFile->file parse"), System.currentTimeMillis() - startMarker); if (parsedResult.isEmpty()) { final FileValidationResult validationResult = new FileValidationResult(); validationResult .addError(new ValidationMessage("Invalid File type", ValidationMessageType.FATAL_ERROR)); returnList.add(validationResult); return returnList; } startMarker = System.currentTimeMillis(); ValidationUtils.invokeValidator(this.fileUploadValidator, parsedResult.getParsedObject(), result); this.metricClient.sendPerformanceMetricToMna(buildMetricMessage(gridFsId, "validateFile->validation"), System.currentTimeMillis() - startMarker); if (result.hasErrors()) { Integer errorCountThresh = null; try { errorCountThresh = Integer.parseInt(this.errorCountThreshold); } catch (final NumberFormatException e) { errorCountThresh = DEFAULT_ERROR_THRESHOLD; } returnList.addAll(ValidationHelper.transform(result, errorCountThresh)); } this.metricClient.sendPerformanceMetricToMna(buildMetricMessage(gridFsId, "validateFile->total time"), System.currentTimeMillis() - start); return returnList; } /** * Preview File. * * @param gridFsId * file to be retrieved for preview. * @return List<FilePreview> file preview. */ @ResponseStatus(HttpStatus.OK) @RequestMapping(value = "/previewFile/{gridFsId}", method = RequestMethod.GET, produces = { MediaType.APPLICATION_JSON_VALUE }) @Secured({ "ROLE_Accommodations Upload", "ROLE_Student Upload", "ROLE_Entity Upload", "ROLE_StudentGroup Upload", "ROLE_User Upload", "ROLE_ExplicitEligibility Upload" }) @ResponseBody public List<FilePreview> previewFile(@PathVariable final String gridFsId) throws Exception { final long start = System.currentTimeMillis(); final GridFSDBFile file = getGridFSDBFile(gridFsId); this.metricClient.sendPerformanceMetricToMna(buildMetricMessage(gridFsId, "previewFile->getGridFSDBFile"), System.currentTimeMillis() - start); return FilePreviewHelper.extractDataForPreview(file.getInputStream(), retrieveFormatTypeFromFileMetadata(file), file.getFilename(), 7); } @ResponseStatus(HttpStatus.OK) @RequestMapping(value = "/saveEntity/{fileId}", method = RequestMethod.GET, produces = { MediaType.APPLICATION_JSON_VALUE }) @Secured({ "ROLE_Accommodations Upload", "ROLE_Student Upload", "ROLE_Entity Upload", "ROLE_StudentGroup Upload", "ROLE_User Upload", "ROLE_ExplicitEligibility Upload" }) @ResponseBody public List<FileUploadSummary> saveEntity(@ModelAttribute("fileId") final String fileId, final BindingResult result, final HttpServletResponse response) throws Exception { final long start = System.currentTimeMillis(); final GridFSDBFile file = getGridFSDBFile(fileId); this.metricClient.sendPerformanceMetricToMna(buildMetricMessage(fileId, "saveEntity->getGridFSDBFile"), System.currentTimeMillis() - start); // reset our start marker... long startMarker = System.currentTimeMillis(); final List<FileUploadSummary> summaryResponse = new ArrayList<>(); //based on FormatType save procedure for DESIGNATEDSUPPORTSANDACCOMMODATIONS is changed if (retrieveFormatTypeFromFileMetadata(file) .equals(FormatType.DESIGNATEDSUPPORTSANDACCOMMODATIONS.name())) { final FileUploadSummary summaryCount = extractAccommodationData(file); summaryResponse.add(summaryCount); } else { final ParserResult<Map<FormatType, List<TestRegistrationBase>>> parsedEntities = extractFile(file); // EDIT: validation during the save step is being skipped due to very low likelihood underlying reference data would be changed between validation step & save step // validate(fileId, result, response); // validate the file // this.metricClient.sendPerformanceMetricToMna("saveEntity->validate", System.currentTimeMillis() - startMarker); for (final Map.Entry<FormatType, List<TestRegistrationBase>> mapEntry : parsedEntities.getParsedObject() .entrySet()) { final FormatType type = mapEntry.getKey(); final List<TestRegistrationBase> testRegistrationBaseList = mapEntry.getValue(); this.metricClient.sendPerformanceMetricToMna( buildMetricMessage(fileId, "saveEntity->processing sb11Entity: type=" + type + "", true), testRegistrationBaseList.size()); // Filter entities that needed to be saved. startMarker = System.currentTimeMillis(); filterErrorRecords(type, parsedEntities.getIgnoredRowNumbers(), testRegistrationBaseList, result.getFieldErrors()); this.metricClient.sendPerformanceMetricToMna( buildMetricMessage(fileId, "saveEntity->filterErrorRecords"), System.currentTimeMillis() - startMarker); // reset our start marker... startMarker = System.currentTimeMillis(); this.sb11DependencyResolverInvoker.resolveDependency(testRegistrationBaseList); // Resolve dependencies if any this.metricClient.sendPerformanceMetricToMna( buildMetricMessage(fileId, "saveEntity->resolveDependency"), System.currentTimeMillis() - startMarker); // reset our start marker... startMarker = System.currentTimeMillis(); final FileUploadSummary summaryCount = this.fileUploadUtils .processGroupEntities(testRegistrationBaseList, this.entityService); this.metricClient.sendPerformanceMetricToMna( buildMetricMessage(fileId, "saveEntity->processGroupEntities"), System.currentTimeMillis() - startMarker); summaryCount.setFormatType(type); summaryResponse.add(summaryCount); } } this.metricClient.sendPerformanceMetricToMna(buildMetricMessage(fileId, "saveEntity->total time"), System.currentTimeMillis() - start); return summaryResponse; } // ===================================================================================================================== /** * @param file * @return */ private FileUploadSummary extractAccommodationData(GridFSDBFile file) throws Exception { return this.extractAccommodationFile(file.getFilename(), file.getInputStream(), retrieveFormatTypeFromFileMetadata(file)); } /** * @param filename * @param inputStream * @return */ private FileUploadSummary extractAccommodationFile(String fileName, InputStream uploadFile, String formatType) throws Exception { final String extension = FilenameUtils.getExtension(fileName); if (extension == null) { throw createNullExtensionException(extension); } switch (FileType.findByFilename(fileName)) { case XLS: case XLSX: return parseExcelFile(XLSX, formatType, uploadFile); case CSV: case TXT: return parseCSVFiles(TXT, formatType, uploadFile); default: throw createFileExtensionException("File extension must be either XLS or XLSX or CSV or TXT"); } } /** * @param txt * @param formatType * @param uploadFile * @return */ private FileUploadSummary parseCSVFiles(FileType txt, String formatType, InputStream uploadFile) throws Exception { final FileUploadSummary fileUploadSummary = new FileUploadSummary(); final Map<String, List<String>> convertedMap = new HashMap<String, List<String>>(); final List<String> excelRows = new ArrayList<String>(); final List<Integer> ignoredRows = new ArrayList<>(); textFileUtils.processTextFile(uploadFile, formatType, new LineMapper() { int totalHeaders; // Regular expression ensures that a split occurs only at commas which are followed by an even (or zero) number of quotes final static String splitRegEx = "(\\t)|(,)(?=([^\"]*\"[^\"]*\")*[^\"]*$)"; @Override public boolean mapLine(final String line, final int lineNumber, final String formatType) { if (lineNumber == HEADER_ROW) { this.totalHeaders = line.trim().split(splitRegEx).length; } else { final String[] columnValues = line.trim().split(splitRegEx); for (int i = 0; i < columnValues.length; i++) { columnValues[i] = FilePreviewHelper.removeExtraneousQuotes(columnValues[i]); } if (!areAllElementsNull(columnValues)) { // ignore rows where all parsed values are blank try { excelRows.add(String.valueOf(fileUploadUtils.entityAccommodationType( padEmptyIfNoColumnAtEnd(this.totalHeaders, trimRecords(columnValues))))); } catch (final Exception e) { throw new RuntimeException(e); } } else { ignoredRows.add(lineNumber); } } convertedMap.put(formatType, excelRows); return true; } }); int count = 0; for (String s : excelRows) { count += Integer.parseInt(s); } fileUploadSummary.setAddedRecords(count); fileUploadSummary.setUpdatedRecords(count); fileUploadSummary.setFormatType(FormatType.DESIGNATEDSUPPORTSANDACCOMMODATIONS); return fileUploadSummary; } /** * @param xlsx * @param formatType * @param uploadFile * @return */ private FileUploadSummary parseExcelFile(FileType xlsx, final String formatType, InputStream uploadFile) throws Exception { final List<Integer> ignoredRowNums = new ArrayList<>(); final FileUploadSummary fileUploadSummary = new FileUploadSummary(); final Map<String, List<String>> convertedMap = new HashMap<String, List<String>>(); final List<String> excelRows = new ArrayList<String>(); this.excelUtils.processExcelFile(uploadFile, new ExcelWorksheetProcessor() { @Override public void process(final Sheet sheet) { // make formattype an array of size 1 to get around "final" access from the inner class final boolean skipHeader = false; excelUtils.iterateRows(sheet, new ExcelRowMapper() { int totalHeaders; @Override public boolean mapRow(final Row row) { try { final String[] records = excelUtils.getRecordsWithNullRowsAsBlank(row); if (getRowNum(row) == HEADER_ROW) { this.totalHeaders = records.length; return true; } if (!isEmptyRecord(records)) { excelRows.add(String.valueOf(fileUploadUtils.entityAccommodationType( padEmptyIfNoColumnAtEnd(this.totalHeaders, trimRecords(records))))); } else { ignoredRowNums.add(getRowNum(row)); } } catch (final Exception e) { throw new RuntimeException(e); } return true; // Continue mapping the row } }, skipHeader, false); convertedMap.put(formatType, excelRows); } }); int count = 0; for (String s : excelRows) { count += Integer.parseInt(s); } fileUploadSummary.setAddedRecords(count); fileUploadSummary.setUpdatedRecords(count); fileUploadSummary.setFormatType(FormatType.DESIGNATEDSUPPORTSANDACCOMMODATIONS); return fileUploadSummary; } private LocalizedException createFileExtensionException(final String extension) { return new LocalizedException("file.invlaid.fileformat", new String[] { extension }); } private LocalizedException createNullExtensionException(final String extension) { return new LocalizedException("file.extension.null", new String[] { extension }); } private GridFSDBFile getGridFSDBFile(final String gridFsId) throws Exception { final GridFSDBFile file = this.fileUploadService.getFileById(gridFsId); if (null == file) { throw new RestException("file.invalid.fileId"); } return file; } private ParserResult<Map<FormatType, List<TestRegistrationBase>>> extractFile(final GridFSDBFile file) throws Exception { return this.fileUploadUtils.extractFile(file.getFilename(), file.getInputStream(), retrieveFormatTypeFromFileMetadata(file)); } private void filterErrorRecords(final FormatType formatType, final List<Integer> ignoredRows, final List<TestRegistrationBase> recordList, final List<FieldError> fieldErrors) { final List<TestRegistrationBase> errorRecords = new ArrayList<>(); for (final FieldError fieldError : fieldErrors) { if (FormatType.valueOf(fieldError.getObjectName()).equals(formatType)) { /* * Each field error has a metadata about where the error occurred in the upload file. Get the row number * and adjust that with the row offset value. Find all error records in the original collection this * way. */ // HEADER ROW + ARRAYLIST ROW INDEX OFFSET + TOTAL IGNORED ROWS final int ROW_OFFSET = 2; final int ignoredRowsSize = ignoredRows.isEmpty() ? 0 : ignoredRows.size(); final int recordRowNumber = ((RowMetadata) fieldError.getArguments()[0]).getRowNum(); if (!CollectionUtils.isEmpty(ignoredRows) && recordRowNumber < ignoredRows.get(0)) { errorRecords.add(recordList.get(recordRowNumber - ROW_OFFSET)); } else { errorRecords.add(recordList.get(recordRowNumber - (ROW_OFFSET + ignoredRowsSize))); } } } recordList.removeAll(errorRecords); } private String retrieveFormatTypeFromFileMetadata(final GridFSDBFile file) { return (String) file.getMetaData().get("formatType"); } private String buildMetricMessage(final String fileId, final String message) { return buildMetricMessage(fileId, message, false); } private String buildMetricMessage(final String fileId, final String message, final boolean notATiming) { return fileId + " " + message + (notATiming ? "" : " (ms)"); } }