Java tutorial
/*- * #%L * %% * Copyright (C) 2005 - 2019 Kuali, Inc. - All Rights Reserved * %% * You may use and modify this code under the terms of the Kuali, Inc. * Pre-Release License Agreement. You may not distribute it. * * You should have received a copy of the Kuali, Inc. Pre-Release License * Agreement with this file. If not, please write to license@kuali.co. * #L% */ package co.kuali.rice.krad.service.impl; import co.kuali.rice.coreservice.api.attachment.RiceAttachmentDataS3Constants; import co.kuali.rice.coreservice.api.attachment.S3FileService; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.kuali.rice.core.api.config.property.ConfigurationService; import org.kuali.rice.coreservice.framework.parameter.ParameterService; import org.kuali.rice.krad.bo.Attachment; import org.kuali.rice.krad.data.DataObjectService; import org.kuali.rice.krad.util.KRADConstants; import org.quartz.DisallowConcurrentExecution; import java.io.*; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Collection; import java.util.Objects; @DisallowConcurrentExecution public class RiceAttachmentDataToS3ConversionImpl implements RiceAttachmentDataToS3Conversion { private static final int MAX_DIR_LEVELS = 6; private static final String DELETE_FILE_FROM_FILESYSTEM = "DELETE_FILE_FROM_FILESYSTEM"; private static final Logger LOG = LogManager.getLogger(RiceAttachmentDataToS3ConversionImpl.class); private S3FileService riceS3FileService; private ParameterService parameterService; private DataObjectService dataObjectService; private ConfigurationService kualiConfigurationService; @Override public void execute() { LOG.info("Starting attachment conversion job for file_data to S3"); if (!processRecords()) { return; } final Collection<Attachment> attachments = dataObjectService.findAll(Attachment.class).getResults(); attachments.forEach(attachment -> { try { final File file = new File(getDocumentDirectory(attachment.getNote().getRemoteObjectIdentifier()) + File.separator + attachment.getAttachmentIdentifier()); if (file.isFile() && file.exists()) { final byte[] fsBytes = FileUtils.readFileToByteArray(file); String fileDataId = attachment.getAttachmentIdentifier(); final Object s3File = riceS3FileService.retrieveFile(fileDataId); final byte[] s3Bytes; if (s3File == null) { final Class<?> s3FileClass = Class.forName(RiceAttachmentDataS3Constants.S3_FILE_CLASS); final Object newS3File = s3FileClass.newInstance(); final Method setId = s3FileClass.getMethod(RiceAttachmentDataS3Constants.SET_ID_METHOD, String.class); setId.invoke(newS3File, fileDataId); final Method setFileContents = s3FileClass.getMethod( RiceAttachmentDataS3Constants.SET_FILE_CONTENTS_METHOD, InputStream.class); try (InputStream stream = new BufferedInputStream(new ByteArrayInputStream(fsBytes))) { setFileContents.invoke(newS3File, stream); riceS3FileService.createFile(newS3File); } s3Bytes = getBytesFromS3File(riceS3FileService.retrieveFile(fileDataId)); } else { if (LOG.isDebugEnabled()) { final Method getFileMetaData = s3File.getClass() .getMethod(RiceAttachmentDataS3Constants.GET_FILE_META_DATA_METHOD); LOG.debug("data found in S3, existing id: " + fileDataId + " note id " + attachment.getNoteIdentifier() + " metadata: " + getFileMetaData.invoke(s3File)); } s3Bytes = getBytesFromS3File(s3File); } if (s3Bytes != null && fsBytes != null) { final String s3MD5 = DigestUtils.md5Hex(s3Bytes); final String dbMD5 = DigestUtils.md5Hex(fsBytes); if (!Objects.equals(s3MD5, dbMD5)) { LOG.error("S3 data MD5: " + s3MD5 + " does not equal DB data MD5: " + dbMD5 + " for id: " + fileDataId + " note id " + attachment.getNoteIdentifier()); } else { if (isDeleteFromFileSystem()) { file.delete(); } } } } } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException | IOException | ClassNotFoundException | InstantiationException e) { throw new RuntimeException(e); } }); LOG.info("Finishing attachment conversion job for file_data to S3"); } protected byte[] getBytesFromS3File(Object s3File) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, IOException { byte[] s3Bytes; final Method getFileContents = s3File.getClass() .getMethod(RiceAttachmentDataS3Constants.GET_FILE_CONTENTS_METHOD); final InputStream fileContents = (InputStream) getFileContents.invoke(s3File); s3Bytes = IOUtils.toByteArray(fileContents); return s3Bytes; } protected String getDocumentDirectory(String objectId) { // Create a directory; all ancestor directories must exist File documentDirectory = new File(getDocumentFileStorageLocation(objectId)); if (!documentDirectory.exists()) { boolean success = documentDirectory.mkdirs(); if (!success) { throw new RuntimeException( "Could not generate directory for File at: " + documentDirectory.getAbsolutePath()); } } return documentDirectory.getAbsolutePath(); } private String getDocumentFileStorageLocation(String objectId) { final String location; if (StringUtils.isEmpty(objectId)) { location = kualiConfigurationService .getPropertyValueAsString(KRADConstants.ATTACHMENTS_PENDING_DIRECTORY_KEY); } else { /* * We need to create a hierarchical directory structure to store * attachment directories, as most file systems max out at 16k * or 32k entries. If we use 6 levels of hierarchy, it allows * hundreds of billions of attachment directories. */ char[] chars = objectId.toUpperCase().replace(" ", "").toCharArray(); int count = chars.length < MAX_DIR_LEVELS ? chars.length : MAX_DIR_LEVELS; StringBuffer prefix = new StringBuffer(); for (int i = 0; i < count; i++) { prefix.append(File.separator + chars[i]); } location = kualiConfigurationService.getPropertyValueAsString(KRADConstants.ATTACHMENTS_DIRECTORY_KEY) + prefix + File.separator + objectId; } return location; } protected boolean processRecords() { final boolean s3IntegrationEnabled = isS3IntegrationEnabled(); if (!s3IntegrationEnabled) { LOG.info("S3 integration is not enabled. Records will not be processed"); } final boolean s3DualSaveEnabled = isS3DualSaveEnabled(); if (s3DualSaveEnabled) { LOG.info("S3 dual save is enabled. Records will not be processed"); } return s3IntegrationEnabled && !s3DualSaveEnabled; } protected boolean isS3IntegrationEnabled() { if (parameterService.parameterExists(KRADConstants.KUALI_RICE_SYSTEM_NAMESPACE, KRADConstants.DetailTypes.ALL_DETAIL_TYPE, RiceAttachmentDataS3Constants.S3_INTEGRATION_ENABLED)) { return parameterService.getParameterValueAsBoolean(KRADConstants.KUALI_RICE_SYSTEM_NAMESPACE, KRADConstants.DetailTypes.ALL_DETAIL_TYPE, RiceAttachmentDataS3Constants.S3_INTEGRATION_ENABLED); } else { return false; } } protected boolean isS3DualSaveEnabled() { return parameterService.getParameterValueAsBoolean(KRADConstants.KUALI_RICE_SYSTEM_NAMESPACE, KRADConstants.DetailTypes.ALL_DETAIL_TYPE, RiceAttachmentDataS3Constants.S3_DUAL_SAVE_ENABLED); } protected boolean isDeleteFromFileSystem() { return parameterService.getParameterValueAsBoolean(KRADConstants.KUALI_RICE_SYSTEM_NAMESPACE, KRADConstants.DetailTypes.ALL_DETAIL_TYPE, DELETE_FILE_FROM_FILESYSTEM); } public S3FileService getRiceS3FileService() { return riceS3FileService; } public void setRiceS3FileService(S3FileService riceS3FileService) { this.riceS3FileService = riceS3FileService; } public ParameterService getParameterService() { return parameterService; } public void setParameterService(ParameterService parameterService) { this.parameterService = parameterService; } public DataObjectService getDataObjectService() { return dataObjectService; } public void setDataObjectService(DataObjectService dataObjectService) { this.dataObjectService = dataObjectService; } public ConfigurationService getKualiConfigurationService() { return kualiConfigurationService; } public void setKualiConfigurationService(ConfigurationService kualiConfigurationService) { this.kualiConfigurationService = kualiConfigurationService; } }