Java tutorial
/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.facebook.presto.hive.s3; import com.amazonaws.AmazonWebServiceClient; import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.InstanceProfileCredentialsProvider; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.AmazonS3EncryptionClient; import com.amazonaws.services.s3.S3ClientOptions; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.EncryptionMaterials; import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; import com.facebook.presto.hive.s3.PrestoS3FileSystem.UnrecoverableS3OperationException; import com.google.common.base.VerifyException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; import org.testng.SkipException; import org.testng.annotations.Test; import javax.crypto.spec.SecretKeySpec; import java.io.IOException; import java.lang.reflect.Field; import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Map; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_ACCESS_KEY; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_CREDENTIALS_PROVIDER; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_ENCRYPTION_MATERIALS_PROVIDER; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_ENDPOINT; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_KMS_KEY_ID; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_MAX_BACKOFF_TIME; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_MAX_CLIENT_RETRIES; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_MAX_RETRY_TIME; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_PATH_STYLE_ACCESS; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_PIN_CLIENT_TO_CURRENT_REGION; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_SECRET_KEY; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_SIGNER_TYPE; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_STAGING_DIRECTORY; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_USER_AGENT_PREFIX; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_USER_AGENT_SUFFIX; import static com.facebook.presto.hive.s3.S3ConfigurationUpdater.S3_USE_INSTANCE_CREDENTIALS; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static io.airlift.testing.Assertions.assertInstanceOf; import static java.nio.file.Files.createTempDirectory; import static java.nio.file.Files.createTempFile; import static org.apache.http.HttpStatus.SC_FORBIDDEN; import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR; import static org.apache.http.HttpStatus.SC_NOT_FOUND; import static org.apache.http.HttpStatus.SC_REQUESTED_RANGE_NOT_SATISFIABLE; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; public class TestPrestoS3FileSystem { @Test public void testStaticCredentials() throws Exception { Configuration config = new Configuration(); config.set(S3_ACCESS_KEY, "test_secret_access_key"); config.set(S3_SECRET_KEY, "test_access_key_id"); // the static credentials should be preferred try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { fs.initialize(new URI("s3n://test-bucket/"), config); assertInstanceOf(getAwsCredentialsProvider(fs), AWSStaticCredentialsProvider.class); } } @Test public void testCompatibleStaticCredentials() throws Exception { Configuration config = new Configuration(); config.set(S3_ACCESS_KEY, "test_secret_access_key"); config.set(S3_SECRET_KEY, "test_access_key_id"); config.set(S3_ENDPOINT, "test.example.endpoint.com"); config.set(S3_SIGNER_TYPE, "S3SignerType"); // the static credentials should be preferred try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { fs.initialize(new URI("s3a://test-bucket/"), config); assertInstanceOf(getAwsCredentialsProvider(fs), AWSStaticCredentialsProvider.class); } } @Test(expectedExceptions = VerifyException.class, expectedExceptionsMessageRegExp = "Invalid configuration: either endpoint can be set or S3 client can be pinned to the current region") public void testEndpointWithPinToCurrentRegionConfiguration() throws Exception { Configuration config = new Configuration(); config.set(S3_ENDPOINT, "test.example.endpoint.com"); config.set(S3_PIN_CLIENT_TO_CURRENT_REGION, "true"); try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { fs.initialize(new URI("s3a://test-bucket/"), config); } } @Test public void testInstanceCredentialsEnabled() throws Exception { Configuration config = new Configuration(); // instance credentials are enabled by default try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { fs.initialize(new URI("s3n://test-bucket/"), config); assertInstanceOf(getAwsCredentialsProvider(fs), InstanceProfileCredentialsProvider.class); } } @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "S3 credentials not configured") public void testInstanceCredentialsDisabled() throws Exception { Configuration config = new Configuration(); config.setBoolean(S3_USE_INSTANCE_CREDENTIALS, false); try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { fs.initialize(new URI("s3n://test-bucket/"), config); } } @Test public void testPathStyleAccess() throws Exception { Configuration config = new Configuration(); config.setBoolean(S3_PATH_STYLE_ACCESS, true); try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { fs.initialize(new URI("s3n://test-bucket/"), config); S3ClientOptions clientOptions = getFieldValue(fs.getS3Client(), AmazonS3Client.class, "clientOptions", S3ClientOptions.class); assertTrue(clientOptions.isPathStyleAccess()); } } @Test public void testUnderscoreBucket() throws Exception { Configuration config = new Configuration(); config.setBoolean(S3_PATH_STYLE_ACCESS, true); try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); String expectedBucketName = "test-bucket_underscore"; fs.initialize(new URI("s3n://" + expectedBucketName + "/"), config); fs.setS3Client(s3); fs.getS3ObjectMetadata(new Path("/test/path")); assertEquals(expectedBucketName, s3.getGetObjectMetadataRequest().getBucketName()); } } @SuppressWarnings({ "ResultOfMethodCallIgnored", "OverlyStrongTypeCast", "ConstantConditions" }) @Test public void testReadRetryCounters() throws Exception { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { int maxRetries = 2; MockAmazonS3 s3 = new MockAmazonS3(); s3.setGetObjectHttpErrorCode(SC_INTERNAL_SERVER_ERROR); Configuration configuration = new Configuration(); configuration.set(S3_MAX_BACKOFF_TIME, "1ms"); configuration.set(S3_MAX_RETRY_TIME, "5s"); configuration.setInt(S3_MAX_CLIENT_RETRIES, maxRetries); fs.initialize(new URI("s3n://test-bucket/"), configuration); fs.setS3Client(s3); try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) { inputStream.read(); } catch (Throwable expected) { assertInstanceOf(expected, AmazonS3Exception.class); assertEquals(((AmazonS3Exception) expected).getStatusCode(), SC_INTERNAL_SERVER_ERROR); assertEquals(PrestoS3FileSystem.getFileSystemStats().getReadRetries().getTotalCount(), maxRetries); assertEquals(PrestoS3FileSystem.getFileSystemStats().getGetObjectRetries().getTotalCount(), (maxRetries + 1L) * maxRetries); } } } @SuppressWarnings({ "OverlyStrongTypeCast", "ConstantConditions" }) @Test public void testGetMetadataRetryCounter() { int maxRetries = 2; try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); s3.setGetObjectMetadataHttpCode(SC_INTERNAL_SERVER_ERROR); Configuration configuration = new Configuration(); configuration.set(S3_MAX_BACKOFF_TIME, "1ms"); configuration.set(S3_MAX_RETRY_TIME, "5s"); configuration.setInt(S3_MAX_CLIENT_RETRIES, maxRetries); fs.initialize(new URI("s3n://test-bucket/"), configuration); fs.setS3Client(s3); fs.getS3ObjectMetadata(new Path("s3n://test-bucket/test")); } catch (Throwable expected) { assertInstanceOf(expected, AmazonS3Exception.class); assertEquals(((AmazonS3Exception) expected).getStatusCode(), SC_INTERNAL_SERVER_ERROR); assertEquals(PrestoS3FileSystem.getFileSystemStats().getGetMetadataRetries().getTotalCount(), maxRetries); } } @SuppressWarnings("ResultOfMethodCallIgnored") @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*Failing getObject call with " + SC_NOT_FOUND + ".*") public void testReadNotFound() throws Exception { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); s3.setGetObjectHttpErrorCode(SC_NOT_FOUND); fs.initialize(new URI("s3n://test-bucket/"), new Configuration()); fs.setS3Client(s3); try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) { inputStream.read(); } } } @SuppressWarnings("ResultOfMethodCallIgnored") @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*Failing getObject call with " + SC_FORBIDDEN + ".*") public void testReadForbidden() throws Exception { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); s3.setGetObjectHttpErrorCode(SC_FORBIDDEN); fs.initialize(new URI("s3n://test-bucket/"), new Configuration()); fs.setS3Client(s3); try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) { inputStream.read(); } } } @Test public void testCreateWithNonexistentStagingDirectory() throws Exception { java.nio.file.Path stagingParent = createTempDirectory("test"); java.nio.file.Path staging = Paths.get(stagingParent.toString(), "staging"); // stagingParent = /tmp/testXXX // staging = /tmp/testXXX/staging try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); Configuration conf = new Configuration(); conf.set(S3_STAGING_DIRECTORY, staging.toString()); fs.initialize(new URI("s3n://test-bucket/"), conf); fs.setS3Client(s3); FSDataOutputStream stream = fs.create(new Path("s3n://test-bucket/test")); stream.close(); assertTrue(Files.exists(staging)); } finally { deleteRecursively(stagingParent, ALLOW_INSECURE); } } @Test(expectedExceptions = IOException.class, expectedExceptionsMessageRegExp = "Configured staging path is not a directory: .*") public void testCreateWithStagingDirectoryFile() throws Exception { java.nio.file.Path staging = createTempFile("staging", null); // staging = /tmp/stagingXXX.tmp try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); Configuration conf = new Configuration(); conf.set(S3_STAGING_DIRECTORY, staging.toString()); fs.initialize(new URI("s3n://test-bucket/"), conf); fs.setS3Client(s3); fs.create(new Path("s3n://test-bucket/test")); } finally { Files.deleteIfExists(staging); } } @Test public void testCreateWithStagingDirectorySymlink() throws Exception { java.nio.file.Path staging = createTempDirectory("staging"); java.nio.file.Path link = Paths.get(staging + ".symlink"); // staging = /tmp/stagingXXX // link = /tmp/stagingXXX.symlink -> /tmp/stagingXXX try { try { Files.createSymbolicLink(link, staging); } catch (UnsupportedOperationException e) { throw new SkipException("Filesystem does not support symlinks", e); } try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); Configuration conf = new Configuration(); conf.set(S3_STAGING_DIRECTORY, link.toString()); fs.initialize(new URI("s3n://test-bucket/"), conf); fs.setS3Client(s3); FSDataOutputStream stream = fs.create(new Path("s3n://test-bucket/test")); stream.close(); assertTrue(Files.exists(link)); } } finally { deleteRecursively(link, ALLOW_INSECURE); deleteRecursively(staging, ALLOW_INSECURE); } } @Test public void testReadRequestRangeNotSatisfiable() throws Exception { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); s3.setGetObjectHttpErrorCode(SC_REQUESTED_RANGE_NOT_SATISFIABLE); fs.initialize(new URI("s3n://test-bucket/"), new Configuration()); fs.setS3Client(s3); try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) { assertEquals(inputStream.read(), -1); } } } @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*Failing getObjectMetadata call with " + SC_FORBIDDEN + ".*") public void testGetMetadataForbidden() throws Exception { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); s3.setGetObjectMetadataHttpCode(SC_FORBIDDEN); fs.initialize(new URI("s3n://test-bucket/"), new Configuration()); fs.setS3Client(s3); fs.getS3ObjectMetadata(new Path("s3n://test-bucket/test")); } } @Test public void testGetMetadataNotFound() throws Exception { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); s3.setGetObjectMetadataHttpCode(SC_NOT_FOUND); fs.initialize(new URI("s3n://test-bucket/"), new Configuration()); fs.setS3Client(s3); assertEquals(fs.getS3ObjectMetadata(new Path("s3n://test-bucket/test")), null); } } @Test public void testEncryptionMaterialsProvider() throws Exception { Configuration config = new Configuration(); config.set(S3_ENCRYPTION_MATERIALS_PROVIDER, TestEncryptionMaterialsProvider.class.getName()); try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { fs.initialize(new URI("s3n://test-bucket/"), config); assertInstanceOf(fs.getS3Client(), AmazonS3EncryptionClient.class); } } @Test public void testKMSEncryptionMaterialsProvider() throws Exception { Configuration config = new Configuration(); config.set(S3_KMS_KEY_ID, "test-key-id"); try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { fs.initialize(new URI("s3n://test-bucket/"), config); assertInstanceOf(fs.getS3Client(), AmazonS3EncryptionClient.class); } } @Test(expectedExceptions = UnrecoverableS3OperationException.class, expectedExceptionsMessageRegExp = ".*\\Q (Path: /tmp/test/path)\\E") public void testUnrecoverableS3ExceptionMessage() { throw new UnrecoverableS3OperationException(new Path("/tmp/test/path"), new IOException("test io exception")); } @Test public void testCustomCredentialsProvider() throws Exception { Configuration config = new Configuration(); config.set(S3_USE_INSTANCE_CREDENTIALS, "false"); config.set(S3_CREDENTIALS_PROVIDER, TestCredentialsProvider.class.getName()); try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { fs.initialize(new URI("s3n://test-bucket/"), config); assertInstanceOf(getAwsCredentialsProvider(fs), TestCredentialsProvider.class); } } @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "Error creating an instance of .*") public void testCustomCredentialsClassCannotBeFound() throws Exception { Configuration config = new Configuration(); config.set(S3_USE_INSTANCE_CREDENTIALS, "false"); config.set(S3_CREDENTIALS_PROVIDER, "com.example.DoesNotExist"); try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { fs.initialize(new URI("s3n://test-bucket/"), config); } } @Test public void testUserAgentPrefix() throws Exception { String userAgentPrefix = "agent_prefix"; Configuration config = new Configuration(); config.set(S3_USER_AGENT_PREFIX, userAgentPrefix); try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { fs.initialize(new URI("s3n://test-bucket/"), config); ClientConfiguration clientConfig = getFieldValue(fs.getS3Client(), AmazonWebServiceClient.class, "clientConfiguration", ClientConfiguration.class); assertEquals(clientConfig.getUserAgentSuffix(), S3_USER_AGENT_SUFFIX); assertEquals(clientConfig.getUserAgentPrefix(), userAgentPrefix); } } @Test public void testDefaultS3ClientConfiguration() throws Exception { HiveS3Config defaults = new HiveS3Config(); try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { fs.initialize(new URI("s3n://test-bucket/"), new Configuration()); ClientConfiguration config = getFieldValue(fs.getS3Client(), AmazonWebServiceClient.class, "clientConfiguration", ClientConfiguration.class); assertEquals(config.getMaxErrorRetry(), defaults.getS3MaxErrorRetries()); assertEquals(config.getConnectionTimeout(), defaults.getS3ConnectTimeout().toMillis()); assertEquals(config.getSocketTimeout(), defaults.getS3SocketTimeout().toMillis()); assertEquals(config.getMaxConnections(), defaults.getS3MaxConnections()); assertEquals(config.getUserAgentSuffix(), S3_USER_AGENT_SUFFIX); assertEquals(config.getUserAgentPrefix(), ""); } } private static AWSCredentialsProvider getAwsCredentialsProvider(PrestoS3FileSystem fs) { return getFieldValue(fs.getS3Client(), "awsCredentialsProvider", AWSCredentialsProvider.class); } private static <T> T getFieldValue(Object instance, String name, Class<T> type) { return getFieldValue(instance, instance.getClass(), name, type); } @SuppressWarnings("unchecked") private static <T> T getFieldValue(Object instance, Class<?> clazz, String name, Class<T> type) { try { Field field = clazz.getDeclaredField(name); checkArgument(field.getType() == type, "expected %s but found %s", type, field.getType()); field.setAccessible(true); return (T) field.get(instance); } catch (ReflectiveOperationException e) { throw new RuntimeException(e); } } private static class TestEncryptionMaterialsProvider implements EncryptionMaterialsProvider { private final EncryptionMaterials encryptionMaterials; public TestEncryptionMaterialsProvider() { encryptionMaterials = new EncryptionMaterials(new SecretKeySpec(new byte[] { 1, 2, 3 }, "AES")); } @Override public void refresh() { } @Override public EncryptionMaterials getEncryptionMaterials(Map<String, String> materialsDescription) { return encryptionMaterials; } @Override public EncryptionMaterials getEncryptionMaterials() { return encryptionMaterials; } } private static class TestCredentialsProvider implements AWSCredentialsProvider { @SuppressWarnings("UnusedParameters") public TestCredentialsProvider(URI uri, Configuration conf) { } @Override public AWSCredentials getCredentials() { return null; } @Override public void refresh() { } } }