Java tutorial
/* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright * ownership. Elasticsearch licenses this file to you under * the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.elasticsearch.repositories.s3; import java.util.HashMap; import java.util.Map; import java.util.function.Function; import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.InstanceProfileCredentialsProvider; import com.amazonaws.http.IdleConnectionReaper; import com.amazonaws.internal.StaticCredentialsProvider; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Service { // pkg private for tests static final Setting<String> CLIENT_NAME = new Setting<>("client", "default", Function.identity()); private final Map<String, S3ClientSettings> clientsSettings; private final Map<String, AmazonS3Client> clientsCache = new HashMap<>(); InternalAwsS3Service(Settings settings, Map<String, S3ClientSettings> clientsSettings) { super(settings); this.clientsSettings = clientsSettings; } @Override public synchronized AmazonS3 client(Settings repositorySettings) { String clientName = CLIENT_NAME.get(repositorySettings); AmazonS3Client client = clientsCache.get(clientName); if (client != null) { return client; } S3ClientSettings clientSettings = clientsSettings.get(clientName); if (clientSettings == null) { throw new IllegalArgumentException( "Unknown s3 client name [" + clientName + "]. Existing client configs: " + Strings.collectionToDelimitedString(clientsSettings.keySet(), ",")); } logger.debug("creating S3 client with client_name [{}], endpoint [{}]", clientName, clientSettings.endpoint); AWSCredentialsProvider credentials = buildCredentials(logger, deprecationLogger, clientSettings, repositorySettings); ClientConfiguration configuration = buildConfiguration(clientSettings, repositorySettings); client = new AmazonS3Client(credentials, configuration); if (Strings.hasText(clientSettings.endpoint)) { client.setEndpoint(clientSettings.endpoint); } clientsCache.put(clientName, client); return client; } // pkg private for tests static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings, Settings repositorySettings) { ClientConfiguration clientConfiguration = new ClientConfiguration(); // the response metadata cache is only there for diagnostics purposes, // but can force objects from every response to the old generation. clientConfiguration.setResponseMetadataCacheSize(0); clientConfiguration.setProtocol(clientSettings.protocol); if (Strings.hasText(clientSettings.proxyHost)) { // TODO: remove this leniency, these settings should exist together and be validated clientConfiguration.setProxyHost(clientSettings.proxyHost); clientConfiguration.setProxyPort(clientSettings.proxyPort); clientConfiguration.setProxyUsername(clientSettings.proxyUsername); clientConfiguration.setProxyPassword(clientSettings.proxyPassword); } clientConfiguration.setMaxErrorRetry(clientSettings.maxRetries); clientConfiguration.setUseThrottleRetries(clientSettings.throttleRetries); clientConfiguration.setSocketTimeout(clientSettings.readTimeoutMillis); return clientConfiguration; } // pkg private for tests static AWSCredentialsProvider buildCredentials(Logger logger, DeprecationLogger deprecationLogger, S3ClientSettings clientSettings, Settings repositorySettings) { BasicAWSCredentials credentials = clientSettings.credentials; if (S3Repository.ACCESS_KEY_SETTING.exists(repositorySettings)) { if (S3Repository.SECRET_KEY_SETTING.exists(repositorySettings) == false) { throw new IllegalArgumentException("Repository setting [" + S3Repository.ACCESS_KEY_SETTING.getKey() + " must be accompanied by setting [" + S3Repository.SECRET_KEY_SETTING.getKey() + "]"); } try (SecureString key = S3Repository.ACCESS_KEY_SETTING.get(repositorySettings); SecureString secret = S3Repository.SECRET_KEY_SETTING.get(repositorySettings)) { credentials = new BasicAWSCredentials(key.toString(), secret.toString()); } // backcompat for reading keys out of repository settings deprecationLogger.deprecated("Using s3 access/secret key from repository settings. Instead " + "store these in named clients and the elasticsearch keystore for secure settings."); } else if (S3Repository.SECRET_KEY_SETTING.exists(repositorySettings)) { throw new IllegalArgumentException("Repository setting [" + S3Repository.SECRET_KEY_SETTING.getKey() + " must be accompanied by setting [" + S3Repository.ACCESS_KEY_SETTING.getKey() + "]"); } if (credentials == null) { logger.debug("Using instance profile credentials"); return new PrivilegedInstanceProfileCredentialsProvider(); } else { logger.debug("Using basic key/secret credentials"); return new StaticCredentialsProvider(credentials); } } /** Returns the value for a given setting from the repository, or returns the fallback value. */ private static <T> T getRepoValue(Settings repositorySettings, Setting<T> repositorySetting, T fallback) { if (repositorySetting.exists(repositorySettings)) { return repositorySetting.get(repositorySettings); } return fallback; } @Override protected void doStart() throws ElasticsearchException { } @Override protected void doStop() throws ElasticsearchException { } @Override protected void doClose() throws ElasticsearchException { for (AmazonS3Client client : clientsCache.values()) { client.shutdown(); } // Ensure that IdleConnectionReaper is shutdown IdleConnectionReaper.shutdown(); } static class PrivilegedInstanceProfileCredentialsProvider implements AWSCredentialsProvider { private final InstanceProfileCredentialsProvider credentials; private PrivilegedInstanceProfileCredentialsProvider() { this.credentials = new InstanceProfileCredentialsProvider(); } @Override public AWSCredentials getCredentials() { return SocketAccess.doPrivileged(credentials::getCredentials); } @Override public void refresh() { SocketAccess.doPrivilegedVoid(credentials::refresh); } } }