Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.usergrid.services.assets.data; import com.google.api.services.storage.StorageScopes; import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.HttpTransportOptions; import com.google.cloud.TransportOptions; import com.google.cloud.WriteChannel; import com.google.cloud.storage.*; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.RandomStringUtils; import org.apache.usergrid.persistence.Entity; import org.apache.usergrid.persistence.EntityManager; import org.apache.usergrid.persistence.EntityManagerFactory; import org.apache.usergrid.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; import java.nio.ByteBuffer; import java.security.GeneralSecurityException; import java.util.Map; import java.util.Properties; import java.util.UUID; import java.util.concurrent.atomic.AtomicLong; public class GoogleBinaryStore implements BinaryStore { private static final Logger logger = LoggerFactory.getLogger(GoogleBinaryStore.class); private static final long FIVE_MB = (FileUtils.ONE_MB * 5); private EntityManagerFactory entityManagerFactory; private Properties properties; private String bucketName; private Storage instance = null; private String reposLocation = FileUtils.getTempDirectoryPath(); public GoogleBinaryStore(Properties properties, EntityManagerFactory entityManagerFactory, String reposLocation) throws IOException, GeneralSecurityException { this.entityManagerFactory = entityManagerFactory; this.properties = properties; this.reposLocation = reposLocation; } private synchronized Storage getService() throws IOException, GeneralSecurityException { logger.trace("Getting Google Cloud Storage service"); // leave this here for tests because they do things like manipulate properties dynamically to test invalid values this.bucketName = properties.getProperty("usergrid.binary.bucketname"); if (instance == null) { // Google provides different authentication types which are different based on if the application is // running within GCE(Google Compute Engine) or GAE (Google App Engine). If Usergrid is running in // GCE or GAE, the SDK will automatically authenticate and get access to // cloud storage. Else, the full path to a credential file should be provided in the following environment variable // // GOOGLE_APPLICATION_CREDENTIALS // // The SDK will attempt to load the credential file for a service account. See the following // for more info: https://developers.google.com/identity/protocols/application-default-credentials#howtheywork GoogleCredentials credentials = GoogleCredentials.getApplicationDefault() .createScoped(StorageScopes.all()); final TransportOptions transportOptions = HttpTransportOptions.newBuilder().setConnectTimeout(30000) // in milliseconds .setReadTimeout(30000) // in milliseconds .build(); instance = StorageOptions.newBuilder().setCredentials(credentials).setTransportOptions(transportOptions) .build().getService(); } return instance; } @Override public void write(UUID appId, Entity entity, InputStream inputStream) throws Exception { getService(); final AtomicLong writtenSize = new AtomicLong(); final int chunkSize = 1024; // one KB // determine max size file allowed, default to 50mb long maxSizeBytes = 50 * FileUtils.ONE_MB; String maxSizeMbString = properties.getProperty("usergrid.binary.max-size-mb", "50"); if (StringUtils.isNumeric(maxSizeMbString)) { maxSizeBytes = Long.parseLong(maxSizeMbString) * FileUtils.ONE_MB; } byte[] firstData = new byte[chunkSize]; int firstSize = inputStream.read(firstData); writtenSize.addAndGet(firstSize); // from the first sample chunk, determine the file size final String contentType = AssetMimeHandler.get().getMimeType(entity, firstData); // Convert to the Google Cloud Storage Blob final BlobId blobId = BlobId.of(bucketName, AssetUtils.buildAssetKey(appId, entity)); final BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType(contentType).build(); // always allow files up to 5mb if (maxSizeBytes < 5 * FileUtils.ONE_MB) { maxSizeBytes = 5 * FileUtils.ONE_MB; } EntityManager em = entityManagerFactory.getEntityManager(appId); Map<String, Object> fileMetadata = AssetUtils.getFileMetadata(entity); // directly upload files that are smaller than the chunk size if (writtenSize.get() < chunkSize) { // Upload to Google cloud Storage instance.create(blobInfo, firstData); } else { WriteChannel writer = instance.writer(blobInfo); // write the initial sample data used to determine file type writer.write(ByteBuffer.wrap(firstData, 0, firstData.length)); // start writing remaining chunks from the stream byte[] buffer = new byte[chunkSize]; int limit; while ((limit = inputStream.read(buffer)) >= 0) { writtenSize.addAndGet(limit); if (writtenSize.get() > maxSizeBytes) { try { fileMetadata.put("error", "Asset size is larger than max size of " + maxSizeBytes); em.update(entity); } catch (Exception e) { logger.error("Error updating entity with error message", e); } return; } try { writer.write(ByteBuffer.wrap(buffer, 0, limit)); } catch (Exception ex) { logger.error("Error writing chunk to Google Cloud Storage for asset "); } } writer.close(); } fileMetadata.put(AssetUtils.CONTENT_LENGTH, writtenSize.get()); fileMetadata.put(AssetUtils.LAST_MODIFIED, System.currentTimeMillis()); fileMetadata.put(AssetUtils.E_TAG, RandomStringUtils.randomAlphanumeric(10)); fileMetadata.put(AssetUtils.CONTENT_TYPE, contentType); try { em.update(entity); } catch (Exception e) { throw new IOException("Unable to update entity filedata", e); } } @Override public InputStream read(UUID appId, Entity entity) throws Exception { return read(appId, entity, 0, FIVE_MB); } @Override public InputStream read(UUID appId, Entity entity, long offset, long length) throws Exception { getService(); final byte[] content = instance .readAllBytes(BlobId.of(bucketName, AssetUtils.buildAssetKey(appId, entity))); return new ByteArrayInputStream(content); } @Override public void delete(UUID appId, Entity entity) throws Exception { getService(); final BlobId blobId = BlobId.of(bucketName, AssetUtils.buildAssetKey(appId, entity)); instance.delete(blobId); } }