Example usage for com.amazonaws.services.s3 AmazonS3Client AmazonS3Client

List of usage examples for com.amazonaws.services.s3 AmazonS3Client AmazonS3Client

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.*
  5. AmazonS3Client
  6. AmazonS3Client

Introduction

In this page you can find the example usage for com.amazonaws.services.s3 AmazonS3Client AmazonS3Client.

Prototype

@Deprecated
public AmazonS3Client(AWSCredentialsProvider credentialsProvider, ClientConfiguration clientConfiguration)

Source Link

Document

Constructs a new Amazon S3 client using the specified AWS credentials and client configuration to access Amazon S3.

Usage

From source file:com.forerunnergames.peril.client.assets.S3AssetUpdater.java

License:Open Source License

S3AssetUpdater(final String bucketPath) {
    Arguments.checkIsNotNull(bucketPath, "bucketPath");
    Preconditions.checkIsTrue(AssetSettings.isValidS3BucketPath(bucketPath),
            AssetSettings.VALID_S3_BUCKET_PATH_DESCRIPTION);

    bucketName = AssetSettings.getS3BucketName(bucketPath);
    executorService = Executors.newSingleThreadExecutor();
    final ClientConfiguration clientConfig = new ClientConfiguration().withMaxErrorRetry(10)
            .withConnectionTimeout(10_000).withSocketTimeout(10_000).withTcpKeepAlive(true);
    final AmazonS3 s3 = new AmazonS3Client(new ProfileCredentialsProvider(), clientConfig);
    transferManager = new TransferManager(s3);
}

From source file:com.ge.predix.sample.blobstore.connector.spring.BlobstoreServiceConnectorCreator.java

License:Apache License

/**
 * Creates the BlobStore context using S3Client
 *
 * @param serviceInfo Object Store Service Info Object
 * @param serviceConnectorConfig Cloud Foundry Service Connector Configuration
 *
 * @return BlobstoreService Instance of the ObjectStore Service
 *//*from w w w. j  av a  2  s.com*/
@Override
public BlobstoreService create(BlobstoreServiceInfo serviceInfo,
        ServiceConnectorConfig serviceConnectorConfig) {
    log.info("create() invoked with serviceInfo? = " + (serviceInfo == null));
    ClientConfiguration config = new ClientConfiguration();
    config.setProtocol(Protocol.HTTPS);

    S3ClientOptions options = new S3ClientOptions();
    config.setSignerOverride("S3SignerType");

    BasicAWSCredentials creds = new BasicAWSCredentials(serviceInfo.getObjectStoreAccessKey(),
            serviceInfo.getObjectStoreSecretKey());
    AmazonS3Client s3Client = new AmazonS3Client(creds, config);
    s3Client.setEndpoint(serviceInfo.getUrl());
    s3Client.setS3ClientOptions(options);

    try {
        // Remove the Credentials from the Object Store URL
        URL url = new URL(serviceInfo.getUrl());
        String urlWithoutCredentials = url.getProtocol() + "://" + url.getHost();

        // Return BlobstoreService
        return new BlobstoreService(s3Client, serviceInfo.getBucket(), urlWithoutCredentials);
    } catch (MalformedURLException e) {
        log.error("create(): Couldnt parse the URL provided by VCAP_SERVICES. Exception = " + e.getMessage());
        throw new RuntimeException("Blobstore URL is Invalid", e);
    }
}

From source file:com.ge.predix.solsvc.blobstore.bootstrap.BlobstoreClientImpl.java

License:Apache License

/**
 *  -//from w w w.  ja va 2 s . com
 */
@PostConstruct
public void init() {

    ClientConfiguration config = new ClientConfiguration();
    config.setProtocol(Protocol.HTTPS);
    if (this.blobstoreConfig.getProxyHost() != null && !"".equals(this.blobstoreConfig.getProxyHost())) { //$NON-NLS-1$
        this.log.info("Connnecting with proxy"); //$NON-NLS-1$
        if (this.blobstoreConfig.getProxyHost() != null) {
            config.withProxyHost(this.blobstoreConfig.getProxyHost());
        }
        if (this.blobstoreConfig.getProxyPort() != null) {
            config.withProxyPort(Integer.parseInt(this.blobstoreConfig.getProxyPort()));
        }
    }
    BasicAWSCredentials creds = new BasicAWSCredentials(this.blobstoreConfig.getAccessKey(),
            this.blobstoreConfig.getAccessKey());
    this.s3Client = new AmazonS3Client(creds, config);
    this.s3Client.setEndpoint(this.blobstoreConfig.getUrl());

}

From source file:com.hpe.caf.worker.datastore.s3.S3DataStore.java

License:Apache License

public S3DataStore(final S3DataStoreConfiguration s3DataStoreConfiguration) {
    if (s3DataStoreConfiguration == null) {
        throw new ArgumentException("s3DataStoreConfiguration was null.");
    }// w  w  w  . j  a v  a 2s .  co m

    ClientConfiguration clientCfg = new ClientConfiguration();

    if (!StringUtils.isNullOrEmpty(s3DataStoreConfiguration.getProxyHost())) {
        clientCfg.setProtocol(Protocol.valueOf(s3DataStoreConfiguration.getProxyProtocol()));
        clientCfg.setProxyHost(s3DataStoreConfiguration.getProxyHost());
        clientCfg.setProxyPort(s3DataStoreConfiguration.getProxyPort());
    }
    AWSCredentials credentials = new BasicAWSCredentials(s3DataStoreConfiguration.getAccessKey(),
            s3DataStoreConfiguration.getSecretKey());
    bucketName = s3DataStoreConfiguration.getBucketName();
    amazonS3Client = new AmazonS3Client(credentials, clientCfg);
    amazonS3Client.setBucketAccelerateConfiguration(new SetBucketAccelerateConfigurationRequest(bucketName,
            new BucketAccelerateConfiguration(BucketAccelerateStatus.Enabled)));
}

From source file:com.ibm.stocator.fs.cos.COSAPIClient.java

License:Apache License

@Override
public void initiate(String scheme) throws IOException, ConfigurationParseException {
    mCachedSparkOriginated = new HashMap<String, Boolean>();
    mCachedSparkJobsStatus = new HashMap<String, Boolean>();
    schemaProvided = scheme;/*from   www.  java2s .c om*/
    Properties props = ConfigurationHandler.initialize(filesystemURI, conf, scheme);
    // Set bucket name property
    int cacheSize = conf.getInt(CACHE_SIZE, GUAVA_CACHE_SIZE_DEFAULT);
    memoryCache = MemoryCache.getInstance(cacheSize);
    mBucket = props.getProperty(COS_BUCKET_PROPERTY);
    workingDir = new Path("/user", System.getProperty("user.name")).makeQualified(filesystemURI,
            getWorkingDirectory());

    fModeAutomaticDelete = "true".equals(props.getProperty(FMODE_AUTOMATIC_DELETE_COS_PROPERTY, "false"));
    mIsV2Signer = "true".equals(props.getProperty(V2_SIGNER_TYPE_COS_PROPERTY, "false"));
    // Define COS client
    String accessKey = props.getProperty(ACCESS_KEY_COS_PROPERTY);
    String secretKey = props.getProperty(SECRET_KEY_COS_PROPERTY);

    if (accessKey == null) {
        throw new ConfigurationParseException("Access KEY is empty. Please provide valid access key");
    }
    if (secretKey == null) {
        throw new ConfigurationParseException("Secret KEY is empty. Please provide valid secret key");
    }

    BasicAWSCredentials creds = new BasicAWSCredentials(accessKey, secretKey);
    ClientConfiguration clientConf = new ClientConfiguration();

    int maxThreads = Utils.getInt(conf, FS_COS, FS_ALT_KEYS, MAX_THREADS, DEFAULT_MAX_THREADS);
    if (maxThreads < 2) {
        LOG.warn(MAX_THREADS + " must be at least 2: forcing to 2.");
        maxThreads = 2;
    }
    int totalTasks = Utils.getInt(conf, FS_COS, FS_ALT_KEYS, MAX_TOTAL_TASKS, DEFAULT_MAX_TOTAL_TASKS);
    long keepAliveTime = Utils.getLong(conf, FS_COS, FS_ALT_KEYS, KEEPALIVE_TIME, DEFAULT_KEEPALIVE_TIME);
    threadPoolExecutor = BlockingThreadPoolExecutorService.newInstance(maxThreads, maxThreads + totalTasks,
            keepAliveTime, TimeUnit.SECONDS, "s3a-transfer-shared");

    unboundedThreadPool = new ThreadPoolExecutor(maxThreads, Integer.MAX_VALUE, keepAliveTime, TimeUnit.SECONDS,
            new LinkedBlockingQueue<Runnable>(),
            BlockingThreadPoolExecutorService.newDaemonThreadFactory("s3a-transfer-unbounded"));

    boolean secureConnections = Utils.getBoolean(conf, FS_COS, FS_ALT_KEYS, SECURE_CONNECTIONS,
            DEFAULT_SECURE_CONNECTIONS);
    clientConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);

    String proxyHost = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, PROXY_HOST, "");
    int proxyPort = Utils.getInt(conf, FS_COS, FS_ALT_KEYS, PROXY_PORT, -1);
    if (!proxyHost.isEmpty()) {
        clientConf.setProxyHost(proxyHost);
        if (proxyPort >= 0) {
            clientConf.setProxyPort(proxyPort);
        } else {
            if (secureConnections) {
                LOG.warn("Proxy host set without port. Using HTTPS default 443");
                clientConf.setProxyPort(443);
            } else {
                LOG.warn("Proxy host set without port. Using HTTP default 80");
                clientConf.setProxyPort(80);
            }
        }
        String proxyUsername = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, PROXY_USERNAME);
        String proxyPassword = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, PROXY_PASSWORD);
        if ((proxyUsername == null) != (proxyPassword == null)) {
            String msg = "Proxy error: " + PROXY_USERNAME + " or " + PROXY_PASSWORD + " set without the other.";
            LOG.error(msg);
            throw new IllegalArgumentException(msg);
        }
        clientConf.setProxyUsername(proxyUsername);
        clientConf.setProxyPassword(proxyPassword);
        clientConf.setProxyDomain(Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, PROXY_DOMAIN));
        clientConf.setProxyWorkstation(Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, PROXY_WORKSTATION));
        if (LOG.isDebugEnabled()) {
            LOG.debug(
                    "Using proxy server {}:{} as user {} with password {} on " + "domain {} as workstation {}",
                    clientConf.getProxyHost(), clientConf.getProxyPort(),
                    String.valueOf(clientConf.getProxyUsername()), clientConf.getProxyPassword(),
                    clientConf.getProxyDomain(), clientConf.getProxyWorkstation());
        }
    } else if (proxyPort >= 0) {
        String msg = "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST;
        LOG.error(msg);
        throw new IllegalArgumentException(msg);
    }

    initConnectionSettings(conf, clientConf);
    if (mIsV2Signer) {
        clientConf.withSignerOverride("S3SignerType");
    }
    mClient = new AmazonS3Client(creds, clientConf);

    final String serviceUrl = props.getProperty(ENDPOINT_URL_COS_PROPERTY);
    if (serviceUrl != null && !serviceUrl.equals(amazonDefaultEndpoint)) {
        mClient.setEndpoint(serviceUrl);
    }
    mClient.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());

    // Set block size property
    String mBlockSizeString = props.getProperty(BLOCK_SIZE_COS_PROPERTY, "128");
    mBlockSize = Long.valueOf(mBlockSizeString).longValue() * 1024 * 1024L;

    boolean autoCreateBucket = "true"
            .equalsIgnoreCase((props.getProperty(AUTO_BUCKET_CREATE_COS_PROPERTY, "false")));

    partSize = Utils.getLong(conf, FS_COS, FS_ALT_KEYS, MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE);
    multiPartThreshold = Utils.getLong(conf, FS_COS, FS_ALT_KEYS, MIN_MULTIPART_THRESHOLD,
            DEFAULT_MIN_MULTIPART_THRESHOLD);
    readAhead = Utils.getLong(conf, FS_COS, FS_ALT_KEYS, READAHEAD_RANGE, DEFAULT_READAHEAD_RANGE);
    LOG.debug(READAHEAD_RANGE + ":" + readAhead);
    inputPolicy = COSInputPolicy
            .getPolicy(Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, INPUT_FADVISE, INPUT_FADV_NORMAL));

    initTransferManager();
    maxKeys = Utils.getInt(conf, FS_COS, FS_ALT_KEYS, MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS);
    flatListingFlag = Utils.getBoolean(conf, FS_COS, FS_ALT_KEYS, FLAT_LISTING, DEFAULT_FLAT_LISTING);

    if (autoCreateBucket) {
        try {
            boolean bucketExist = mClient.doesBucketExist(mBucket);
            if (bucketExist) {
                LOG.trace("Bucket {} exists", mBucket);
            } else {
                LOG.trace("Bucket {} doesn`t exists and autocreate", mBucket);
                String mRegion = props.getProperty(REGION_COS_PROPERTY);
                if (mRegion == null) {
                    mClient.createBucket(mBucket);
                } else {
                    LOG.trace("Creating bucket {} in region {}", mBucket, mRegion);
                    mClient.createBucket(mBucket, mRegion);
                }
            }
        } catch (AmazonServiceException ase) {
            /*
            *  we ignore the BucketAlreadyExists exception since multiple processes or threads
            *  might try to create the bucket in parrallel, therefore it is expected that
            *  some will fail to create the bucket
            */
            if (!ase.getErrorCode().equals("BucketAlreadyExists")) {
                LOG.error(ase.getMessage());
                throw (ase);
            }
        } catch (Exception e) {
            LOG.error(e.getMessage());
            throw (e);
        }
    }

    initMultipartUploads(conf);
    enableMultiObjectsDelete = Utils.getBoolean(conf, FS_COS, FS_ALT_KEYS, ENABLE_MULTI_DELETE, true);

    blockUploadEnabled = Utils.getBoolean(conf, FS_COS, FS_ALT_KEYS, FAST_UPLOAD, DEFAULT_FAST_UPLOAD);

    if (blockUploadEnabled) {
        blockOutputBuffer = Utils.getTrimmed(conf, FS_COS, FS_ALT_KEYS, FAST_UPLOAD_BUFFER,
                DEFAULT_FAST_UPLOAD_BUFFER);
        partSize = COSUtils.ensureOutputParameterInRange(MULTIPART_SIZE, partSize);
        blockFactory = COSDataBlocks.createFactory(this, blockOutputBuffer);
        blockOutputActiveBlocks = Utils.getInt(conf, FS_COS, FS_ALT_KEYS, FAST_UPLOAD_ACTIVE_BLOCKS,
                DEFAULT_FAST_UPLOAD_ACTIVE_BLOCKS);
        LOG.debug("Using COSBlockOutputStream with buffer = {}; block={};" + " queue limit={}",
                blockOutputBuffer, partSize, blockOutputActiveBlocks);
    } else {
        LOG.debug("Using COSOutputStream");
    }
}

From source file:com.intuit.tank.vmManager.environment.amazon.AmazonS3.java

License:Open Source License

/**
 * //from w ww  . j  av  a2s .c o m
 */
public AmazonS3() {

    try {
        CloudCredentials creds = new TankConfig().getVmManagerConfig()
                .getCloudCredentials(CloudProvider.amazon);
        ClientConfiguration config = new ClientConfiguration();
        if (StringUtils.isNotBlank(System.getProperty("http.proxyHost"))) {
            try {
                config.setProxyHost(System.getProperty("http.proxyHost"));
                if (StringUtils.isNotBlank(System.getProperty("http.proxyPort"))) {
                    config.setProxyPort(Integer.valueOf(System.getProperty("http.proxyPort")));
                }
            } catch (NumberFormatException e) {
                LOG.error("invalid proxy setup.");
            }

        }
        if (StringUtils.isNotBlank(creds.getKeyId()) && StringUtils.isNotBlank(creds.getKey())) {
            BasicAWSCredentials credentials = new BasicAWSCredentials(creds.getKeyId(), creds.getKey());
            this.s3Client = new AmazonS3Client(credentials, config);
        } else {
            this.s3Client = new AmazonS3Client(config);
        }
    } catch (Exception ex) {
        LOG.error(ex.getMessage());
        throw new RuntimeException(ex);
    }
}

From source file:com.liferay.portal.store.s3.S3Store.java

License:Open Source License

protected AmazonS3 getAmazonS3(AWSCredentialsProvider awsCredentialsProvider) {

    ClientConfiguration clientConfiguration = getClientConfiguration();

    AmazonS3 amazonS3 = new AmazonS3Client(awsCredentialsProvider, clientConfiguration);

    Region region = Region.getRegion(Regions.fromName(_s3StoreConfiguration.s3Region()));

    amazonS3.setRegion(region);//  ww  w .  j a  v  a  2s  . c  om

    return amazonS3;
}

From source file:com.netflix.exhibitor.core.s3.S3ClientImpl.java

License:Apache License

private AmazonS3Client createClient(AWSCredentialsProvider awsCredentialProvider,
        BasicAWSCredentials basicAWSCredentials, S3ClientConfig clientConfig) {
    AmazonS3Client localClient;//from w  w w . j  ava  2s.c o  m
    if (awsCredentialProvider != null) {
        if (clientConfig != null) {
            localClient = new AmazonS3Client(awsCredentialProvider, clientConfig.getAWSClientConfig());
        } else {
            localClient = new AmazonS3Client(awsCredentialProvider);
        }
    } else if (basicAWSCredentials != null) {
        if (clientConfig != null) {
            localClient = new AmazonS3Client(basicAWSCredentials, clientConfig.getAWSClientConfig());
        } else {
            localClient = new AmazonS3Client(basicAWSCredentials);
        }
    } else {
        if (clientConfig != null) {
            localClient = new AmazonS3Client(clientConfig.getAWSClientConfig());
        } else {
            localClient = new AmazonS3Client();
        }
    }

    if (s3Region != null) {
        String fixedRegion = s3Region.equals("us-east-1") ? "" : ("-" + s3Region);
        String endpoint = ENDPOINT_SPEC.replace("$REGION$", fixedRegion);
        localClient.setEndpoint(endpoint);
        log.info("Setting S3 endpoint to: " + endpoint);
    }

    return localClient;
}

From source file:com.netflix.genie.web.configs.aws.AwsS3Config.java

License:Apache License

/**
 * A bean providing a client to work with S3.
 *
 * @param noOfS3Retries          No. of S3 request retries
 * @param awsCredentialsProvider A credentials provider used to instantiate the client.
 * @return An amazon s3 client object/* w ww. j a  va 2  s . c  o m*/
 */
@Bean
@ConditionalOnBean(AWSCredentialsProvider.class)
public AmazonS3Client genieS3Client(@Value("${genie.retry.s3.noOfRetries:5}") final int noOfS3Retries,
        final AWSCredentialsProvider awsCredentialsProvider) {
    final ClientConfiguration clientConfiguration = new ClientConfiguration()
            .withRetryPolicy(PredefinedRetryPolicies.getDefaultRetryPolicyWithCustomMaxRetries(noOfS3Retries));
    return new AmazonS3Client(awsCredentialsProvider, clientConfiguration);
}

From source file:com.netflix.ice.common.AwsUtils.java

License:Apache License

/**
 * This method must be called before all methods can be used.
 * @param credentialsProvider//from  w ww  .  j a v a 2s.c om
 */
public static void init(AWSCredentialsProvider credentialsProvider) {
    awsCredentialsProvider = credentialsProvider;
    clientConfig = new ClientConfiguration();
    String proxyHost = System.getProperty("https.proxyHost");
    String proxyPort = System.getProperty("https.proxyPort");
    if (proxyHost != null && proxyPort != null) {
        clientConfig.setProxyHost(proxyHost);
        clientConfig.setProxyPort(Integer.parseInt(proxyPort));
    }
    s3Client = new AmazonS3Client(awsCredentialsProvider, clientConfig);
    securityClient = new AWSSecurityTokenServiceClient(awsCredentialsProvider, clientConfig);
    if (System.getProperty("EC2_REGION") != null && !"us-east-1".equals(System.getProperty("EC2_REGION"))) {
        if ("global".equals(System.getProperty("EC2_REGION"))) {
            s3Client.setEndpoint("s3.amazonaws.com");
        } else {
            s3Client.setEndpoint("s3-" + System.getProperty("EC2_REGION") + ".amazonaws.com");
        }
    }
}