Java tutorial
/* * Copyright 2015 herd contributors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.finra.herd.service.impl; import java.util.UUID; import com.amazonaws.auth.policy.actions.S3Actions; import com.amazonaws.services.securitytoken.model.Credentials; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.Assert; import org.finra.herd.core.HerdDateUtils; import org.finra.herd.core.helper.ConfigurationHelper; import org.finra.herd.dao.BusinessObjectDataDao; import org.finra.herd.dao.StsDao; import org.finra.herd.dao.config.DaoSpringModuleConfig; import org.finra.herd.dao.helper.AwsHelper; import org.finra.herd.model.AlreadyExistsException; import org.finra.herd.model.annotation.NamespacePermission; import org.finra.herd.model.api.xml.AwsCredential; import org.finra.herd.model.api.xml.BusinessObjectDataKey; import org.finra.herd.model.api.xml.BusinessObjectFormatKey; import org.finra.herd.model.api.xml.NamespacePermissionEnum; import org.finra.herd.model.api.xml.S3KeyPrefixInformation; import org.finra.herd.model.api.xml.StorageUnitDownloadCredential; import org.finra.herd.model.api.xml.StorageUnitUploadCredential; import org.finra.herd.model.dto.ConfigurationValue; import org.finra.herd.model.jpa.BusinessObjectDataEntity; import org.finra.herd.model.jpa.BusinessObjectFormatEntity; import org.finra.herd.model.jpa.StorageEntity; import org.finra.herd.service.StorageUnitService; import org.finra.herd.service.helper.AwsPolicyBuilder; import org.finra.herd.service.helper.BusinessObjectDataHelper; import org.finra.herd.service.helper.BusinessObjectFormatDaoHelper; import org.finra.herd.service.helper.KmsActions; import org.finra.herd.service.helper.S3KeyPrefixHelper; import org.finra.herd.service.helper.StorageDaoHelper; import org.finra.herd.service.helper.StorageHelper; @Service @Transactional(value = DaoSpringModuleConfig.HERD_TRANSACTION_MANAGER_BEAN_NAME) public class StorageUnitServiceImpl implements StorageUnitService { @Autowired private AwsHelper awsHelper; @Autowired private BusinessObjectDataDao businessObjectDataDao; @Autowired private BusinessObjectDataHelper businessObjectDataHelper; @Autowired private BusinessObjectFormatDaoHelper businessObjectFormatDaoHelper; @Autowired private ConfigurationHelper configurationHelper; @Autowired private S3KeyPrefixHelper s3KeyPrefixHelper; @Autowired private StorageDaoHelper storageDaoHelper; @Autowired private StorageHelper storageHelper; @Autowired private StsDao stsDao; @NamespacePermission(fields = "#businessObjectDataKey.namespace", permissions = NamespacePermissionEnum.READ) @Override @Transactional(propagation = Propagation.REQUIRES_NEW) public S3KeyPrefixInformation getS3KeyPrefix(BusinessObjectDataKey businessObjectDataKey, String businessObjectFormatPartitionKey, String storageName, Boolean createNewVersion) { return getS3KeyPrefixImpl(businessObjectDataKey, businessObjectFormatPartitionKey, storageName, createNewVersion); } @NamespacePermission(fields = "#businessObjectDataKey?.namespace", permissions = NamespacePermissionEnum.WRITE) @Override public StorageUnitUploadCredential getStorageUnitUploadCredential(BusinessObjectDataKey businessObjectDataKey, Boolean createNewVersion, String storageName) { StorageUnitUploadCredential businessObjectDataUploadCredential = new StorageUnitUploadCredential(); businessObjectDataUploadCredential.setAwsCredential( getBusinessObjectDataS3Credential(businessObjectDataKey, createNewVersion, storageName, true)); businessObjectDataUploadCredential .setAwsKmsKeyId(getStorageKmsKeyId(storageDaoHelper.getStorageEntity(storageName.trim()))); return businessObjectDataUploadCredential; } @NamespacePermission(fields = "#businessObjectDataKey?.namespace", permissions = NamespacePermissionEnum.READ) @Override public StorageUnitDownloadCredential getStorageUnitDownloadCredential( BusinessObjectDataKey businessObjectDataKey, String storageName) { StorageUnitDownloadCredential businessObjectDataDownloadCredential = new StorageUnitDownloadCredential(); businessObjectDataDownloadCredential.setAwsCredential( getBusinessObjectDataS3Credential(businessObjectDataKey, null, storageName, false)); return businessObjectDataDownloadCredential; } /** * Gets the S3 key prefix. * * @param businessObjectDataKey the business object data key * @param businessObjectFormatPartitionKey the business object format partition key * @param storageName the storage name * @param createNewVersion specifies if it is OK to return an S3 key prefix for a new business object data version that is not an initial version. This * parameter is ignored, when the business object data version is specified. * * @return the S3 key prefix */ protected S3KeyPrefixInformation getS3KeyPrefixImpl(BusinessObjectDataKey businessObjectDataKey, String businessObjectFormatPartitionKey, String storageName, Boolean createNewVersion) { // Validate and trim the business object data key. businessObjectDataHelper.validateBusinessObjectDataKey(businessObjectDataKey, true, false); // If specified, trim the partition key parameter. String businessObjectFormatPartitionKeyLocal = businessObjectFormatPartitionKey; if (businessObjectFormatPartitionKeyLocal != null) { businessObjectFormatPartitionKeyLocal = businessObjectFormatPartitionKeyLocal.trim(); } // If specified, trim the storage name. Otherwise, default to the configuration option. String storageNameLocal = storageName; if (StringUtils.isNotBlank(storageNameLocal)) { storageNameLocal = storageNameLocal.trim(); } else { storageNameLocal = configurationHelper.getProperty(ConfigurationValue.S3_STORAGE_NAME_DEFAULT); } // Get the business object format for the specified parameters and make sure it exists. BusinessObjectFormatEntity businessObjectFormatEntity = businessObjectFormatDaoHelper .getBusinessObjectFormatEntity(new BusinessObjectFormatKey(businessObjectDataKey.getNamespace(), businessObjectDataKey.getBusinessObjectDefinitionName(), businessObjectDataKey.getBusinessObjectFormatUsage(), businessObjectDataKey.getBusinessObjectFormatFileType(), businessObjectDataKey.getBusinessObjectFormatVersion())); // If specified, ensure that partition key matches what's configured within the business object format. if (StringUtils.isNotBlank(businessObjectFormatPartitionKeyLocal)) { Assert.isTrue( businessObjectFormatEntity.getPartitionKey() .equalsIgnoreCase(businessObjectFormatPartitionKeyLocal), "Partition key \"" + businessObjectFormatPartitionKeyLocal + "\" doesn't match configured business object format partition key \"" + businessObjectFormatEntity.getPartitionKey() + "\"."); } // Get and validate the storage along with the relative attributes. StorageEntity storageEntity = storageDaoHelper.getStorageEntity(storageNameLocal); // If the business object data version is not specified, get the next business object data version value. if (businessObjectDataKey.getBusinessObjectDataVersion() == null) { // Get the latest data version for this business object data, if it exists. BusinessObjectDataEntity latestVersionBusinessObjectDataEntity = businessObjectDataDao .getBusinessObjectDataByAltKey(new BusinessObjectDataKey(businessObjectDataKey.getNamespace(), businessObjectDataKey.getBusinessObjectDefinitionName(), businessObjectDataKey.getBusinessObjectFormatUsage(), businessObjectDataKey.getBusinessObjectFormatFileType(), businessObjectDataKey.getBusinessObjectFormatVersion(), businessObjectDataKey.getPartitionValue(), businessObjectDataKey.getSubPartitionValues(), null)); // Throw an error if this business object data already exists and createNewVersion flag is not set. if (latestVersionBusinessObjectDataEntity != null && !createNewVersion) { throw new AlreadyExistsException("Initial version of the business object data already exists."); } businessObjectDataKey.setBusinessObjectDataVersion(latestVersionBusinessObjectDataEntity == null ? BusinessObjectDataEntity.BUSINESS_OBJECT_DATA_INITIAL_VERSION : latestVersionBusinessObjectDataEntity.getVersion() + 1); } // Build the S3 key prefix string. String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(storageEntity, businessObjectFormatEntity, businessObjectDataKey); // Create and return the S3 key prefix. S3KeyPrefixInformation s3KeyPrefixInformation = new S3KeyPrefixInformation(); s3KeyPrefixInformation.setS3KeyPrefix(s3KeyPrefix); return s3KeyPrefixInformation; } /** * Creates and returns a set of AWS credentials which can be used to access the S3 object indicated by the given business object data and storage. * * @param businessObjectDataKey Business object data key * @param createNewVersion true to create credentials for the next version up from the latest business object data, otherwise, uses specified data version * in data key. * @param storageName Name of storage to access * @param isUpload true if this credential is to upload, false to download * * @return Credentials which has the permissions to perform the specified actions at the specified storage. */ private AwsCredential getBusinessObjectDataS3Credential(BusinessObjectDataKey businessObjectDataKey, Boolean createNewVersion, String storageName, boolean isUpload) { Assert.isTrue(StringUtils.isNotBlank(storageName), "storageName must be specified"); Assert.isTrue(businessObjectDataKey.getBusinessObjectDataVersion() != null || createNewVersion != null, "One of businessObjectDataVersion or createNewVersion must be specified."); Assert.isTrue( businessObjectDataKey.getBusinessObjectDataVersion() == null || !Boolean.TRUE.equals(createNewVersion), "createNewVersion must be false or unspecified when businessObjectDataVersion is specified."); /* * Choose configurations based on whether this is an upload or download operation. */ ConfigurationValue roleArnConfigurationValue; ConfigurationValue defaultSessionDurationConfigurationValue; ConfigurationValue sessionDurationConfigurationValue; S3Actions[] s3Actions; KmsActions[] kmsActions; if (isUpload) { roleArnConfigurationValue = ConfigurationValue.S3_ATTRIBUTE_NAME_UPLOAD_ROLE_ARN; defaultSessionDurationConfigurationValue = ConfigurationValue.AWS_S3_DEFAULT_UPLOAD_SESSION_DURATION_SECS; sessionDurationConfigurationValue = ConfigurationValue.S3_ATTRIBUTE_NAME_UPLOAD_SESSION_DURATION_SECS; s3Actions = new S3Actions[] { S3Actions.PutObject, S3Actions.DeleteObject }; kmsActions = new KmsActions[] { KmsActions.GENERATE_DATA_KEY, KmsActions.DECRYPT }; } else { roleArnConfigurationValue = ConfigurationValue.S3_ATTRIBUTE_NAME_DOWNLOAD_ROLE_ARN; defaultSessionDurationConfigurationValue = ConfigurationValue.AWS_S3_DEFAULT_DOWNLOAD_SESSION_DURATION_SECS; sessionDurationConfigurationValue = ConfigurationValue.S3_ATTRIBUTE_NAME_DOWNLOAD_SESSION_DURATION_SECS; s3Actions = new S3Actions[] { S3Actions.GetObject }; kmsActions = new KmsActions[] { KmsActions.DECRYPT }; } StorageEntity storageEntity = storageDaoHelper.getStorageEntity(storageName.trim()); String roleArn = storageHelper.getStorageAttributeValueByName( configurationHelper.getProperty(roleArnConfigurationValue), storageEntity, true); Integer durationSeconds = storageHelper.getStorageAttributeIntegerValueByName( configurationHelper.getProperty(sessionDurationConfigurationValue), storageEntity, configurationHelper.getProperty(defaultSessionDurationConfigurationValue, Integer.class)); String bucketName = storageHelper.getStorageAttributeValueByName( configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storageEntity, true); S3KeyPrefixInformation s3KeyPrefixInformation = getS3KeyPrefixImpl(businessObjectDataKey, null, storageName, createNewVersion); /* * Policy is different based on whether this is meant for downloading or uploading. * However, both uploader and downloader requires a ListBucket at the bucket level. */ AwsPolicyBuilder awsPolicyBuilder = new AwsPolicyBuilder() .withS3Prefix(bucketName, s3KeyPrefixInformation.getS3KeyPrefix(), s3Actions) .withS3(bucketName, null, S3Actions.ListObjects); /* * Only add KMS policies if the storage specifies a KMS ID */ String kmsKeyId = getStorageKmsKeyId(storageEntity); if (kmsKeyId != null) { awsPolicyBuilder.withKms(kmsKeyId.trim(), kmsActions); } Credentials credentials = stsDao.getTemporarySecurityCredentials(awsHelper.getAwsParamsDto(), UUID.randomUUID().toString(), roleArn, durationSeconds, awsPolicyBuilder.build()); AwsCredential awsCredential = new AwsCredential(); awsCredential.setAwsAccessKey(credentials.getAccessKeyId()); awsCredential.setAwsSecretKey(credentials.getSecretAccessKey()); awsCredential.setAwsSessionToken(credentials.getSessionToken()); awsCredential.setAwsSessionExpirationTime( HerdDateUtils.getXMLGregorianCalendarValue(credentials.getExpiration())); return awsCredential; } private String getStorageKmsKeyId(StorageEntity storageEntity) { return storageHelper.getStorageAttributeValueByName( configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KMS_KEY_ID), storageEntity, false, true); } }