Example usage for com.amazonaws.services.s3.model S3Object getObjectContent

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model S3Object getObjectContent.

Prototype

public S3ObjectInputStream getObjectContent() 

Source Link

Document

Gets the input stream containing the contents of this object.

Usage

From source file:org.apache.manifoldcf.crawler.connectors.amazons3.AmazonS3Connector.java

License:Apache License

@Override
public void processDocuments(String[] documentIdentifiers, IExistingVersions statuses, Specification spec,
        IProcessActivity activities, int jobMode, boolean usesDefaultAuthority)
        throws ManifoldCFException, ServiceInterruption {
    AmazonS3 amazons3Client = getClient();
    if (amazons3Client == null)
        throw new ManifoldCFException("Amazon client can not connect at the moment");
    String[] acls = null;//from  w w  w.  j  ava  2 s. c om

    // loop documents and process
    for (String documentIdentifier : documentIdentifiers) {
        try {
            if (documentIdentifier != null && StringUtils.isNotEmpty(documentIdentifier)) {
                String versionString;
                String[] aclsToUse;

                if (documentIdentifier.split(STD_SEPARATOR_BUCKET_AND_KEY) == null
                        && documentIdentifier.length() < 1) {
                    continue;
                }

                S3Artifact s3Artifact = getS3Artifact(documentIdentifier);
                S3Object s3Obj = amazons3Client
                        .getObject(new GetObjectRequest(s3Artifact.getBucketName(), s3Artifact.getKey()));

                if (s3Obj == null) {
                    // no such document in the bucket now
                    // delete document
                    activities.deleteDocument(documentIdentifier);
                    continue;
                }

                Logging.connectors.info("Content-Type: " + s3Obj.getObjectMetadata().getContentType());
                ObjectMetadata objectMetadata = s3Obj.getObjectMetadata();
                Date lastModified = objectMetadata.getLastModified();
                StringBuilder sb = new StringBuilder();
                if (lastModified == null) {
                    // remove the content
                    activities.deleteDocument(documentIdentifier);
                    continue;
                }

                aclsToUse = new String[0];

                AccessControlList objectAcl = amazons3Client.getObjectAcl(s3Artifact.getBucketName(),
                        s3Artifact.getKey());

                Set<Grant> grants = objectAcl.getGrants();
                String[] users = getUsers(grants);
                // sort

                aclsToUse = users;
                Arrays.sort(aclsToUse);
                packList(sb, aclsToUse, '+');
                if (aclsToUse.length > 0) {
                    sb.append('+');
                    pack(sb, AmazonS3Config.defaultAuthorityDenyToken, '+');
                } else
                    sb.append('-');

                //
                sb.append(lastModified.toString());
                versionString = sb.toString();

                Logging.connectors.debug("version string : " + versionString);

                if (versionString.length() > 0
                        && !activities.checkDocumentNeedsReindexing(documentIdentifier, versionString)) {
                    Logging.connectors.info("Document need not to be reindexed : " + documentIdentifier);
                    continue;
                }

                Logging.connectors.debug("JIRA: Processing document identifier '" + documentIdentifier + "'");

                long startTime = System.currentTimeMillis();
                String errorCode = null;
                String errorDesc = null;
                Long fileSize = null;

                try {
                    String mimeType = "text/plain";// default

                    // tika works starts
                    InputStream in = null;

                    String document = null;
                    try {
                        in = s3Obj.getObjectContent();

                        parser.parse(in, handler, metadata, context);
                        mimeType = tika.detect(in);
                        document = handler.toString();
                        if (document == null)
                            continue;
                        metadata.set(Metadata.CONTENT_TYPE, mimeType);
                    } catch (Exception e) {
                        Logging.connectors.error("Error while parsing tika contents", e);
                    } finally {
                        if (in != null)
                            IOUtils.closeQuietly(in);
                    }

                    String documentURI = getDocumentURI(s3Artifact);

                    Logging.connectors.debug("document : " + documentURI);

                    // need some investigation
                    if (!activities.checkURLIndexable(documentURI)) {
                        errorCode = activities.EXCLUDED_URL;
                        errorDesc = "Excluded because of URL ('" + documentURI + "')";
                        activities.noDocument(documentIdentifier, versionString);
                        continue;
                    }
                    if (!activities.checkMimeTypeIndexable(mimeType)) {
                        errorCode = activities.EXCLUDED_MIMETYPE;
                        errorDesc = "Excluded because of mime type ('" + mimeType + "')";
                        activities.noDocument(documentIdentifier, versionString);
                        continue;
                    }
                    if (!activities.checkDateIndexable(lastModified)) {
                        errorCode = activities.EXCLUDED_DATE;
                        errorDesc = "Excluded because of date (" + lastModified + ")";
                        activities.noDocument(documentIdentifier, versionString);
                        continue;
                    }

                    // otherwise process
                    RepositoryDocument rd = new RepositoryDocument();
                    // Turn into acls and add into
                    // description
                    String[] denyAclsToUse;
                    if (aclsToUse.length > 0)
                        denyAclsToUse = new String[] { AmazonS3Config.defaultAuthorityDenyToken };
                    else
                        denyAclsToUse = new String[0];
                    rd.setSecurity(RepositoryDocument.SECURITY_TYPE_DOCUMENT, aclsToUse, denyAclsToUse);

                    rd.setMimeType(mimeType);

                    if (lastModified != null)
                        rd.setModifiedDate(lastModified);

                    // set all meta-data fields
                    addAllMetaData(rd, metadata);

                    // get document

                    try {
                        byte[] documentBytes = document.getBytes(StandardCharsets.UTF_8);
                        long fileLength = documentBytes.length;

                        if (!activities.checkLengthIndexable(fileLength)) {
                            errorCode = activities.EXCLUDED_LENGTH;
                            errorDesc = "Excluded because of document length (" + fileLength + ")";
                            activities.noDocument(documentIdentifier, versionString);
                            continue;
                        }

                        InputStream is = new ByteArrayInputStream(documentBytes);
                        try {
                            rd.setBinary(is, fileLength);
                            activities.ingestDocumentWithException(documentIdentifier, versionString,
                                    documentURI, rd);

                            errorCode = "OK";
                            fileSize = new Long(fileLength);
                        } finally {
                            if (is != null)
                                IOUtils.closeQuietly(is);
                        }
                    } catch (Exception e) {
                        Logging.connectors.error(e);
                    }
                } catch (Exception e) {
                    Logging.connectors.error(e);
                }

            }
        } catch (AmazonServiceException e) {
            Logging.connectors.error(e);
        } catch (AmazonClientException e) {
            Logging.connectors.error(e);
        }

    }

}

---

From source file:org.apache.oodt.cas.filemgr.datatransfer.S3DataTransferer.java

License:Apache License

private void stageFile(S3Object file, Reference ref, File directory) throws IOException {
    S3ObjectInputStream inStream = null;
    FileOutputStream outStream = null;
    try {//from   w w  w .j ava2  s  .  c om
        inStream = file.getObjectContent();
        outStream = new FileOutputStream(
                new File(directory, new File(stripProtocol(ref.getDataStoreReference(), false)).getName()));
        IOUtils.copy(inStream, outStream);
    } finally {
        try {
            inStream.close();
        } catch (Exception ignored) {
        }
        try {
            outStream.close();
        } catch (Exception ignored) {
        }
    }
}

---

From source file:org.apache.usergrid.apm.service.CrashUtil.java

License:Apache License

public static String getCrashSummary(String fullAppName, String s3CrashFileName) {

    DeploymentConfig config = DeploymentConfig.geDeploymentConfig();
    AWSCredentials credentials = new BasicAWSCredentials(config.getAccessKey(), config.getSecretKey());
    AmazonS3Client s3Client = new AmazonS3Client(credentials);
    String s3FullFileName = AWSUtil.formS3CrashFileUrl(fullAppName, s3CrashFileName);

    log.info("Crash file bucket " + config.getS3LogBucket() + " and file name : " + s3FullFileName);

    GetObjectRequest objectRequest = new GetObjectRequest(config.getS3LogBucket(), s3FullFileName);
    String crashSummary = null;/* w  w w .  j  a  v a  2s . co  m*/
    try {
        S3Object s3Object = s3Client.getObject(objectRequest);
        InputStream is = s3Object.getObjectContent();
        String fileContents = IOUtils.toString(is);

        CrashLogParser parser = null;
        if (fileContents != null) {
            if (s3CrashFileName.endsWith(".crash")) {
                parser = new iOSCrashLogParser();
                if (parser.parseCrashLog(fileContents))
                    crashSummary = parser.getCrashSummary();
                else {
                    log.error("problem parsing ios crash file for app " + fullAppName + " file: "
                            + s3CrashFileName);
                    crashSummary = "Not able to get summary for iOS crash log";
                }
            } else if (s3CrashFileName.endsWith(".stacktrace")) {
                parser = new AndroidCrashLogParser();
                if (parser.parseCrashLog(fileContents))
                    crashSummary = parser.getCrashSummary();
                else {
                    log.error("problem parsing Android crash file for app " + fullAppName + " file: "
                            + s3CrashFileName);
                    crashSummary = "Not able to get summary for Android crash log";
                }
            }
        }
    } catch (AmazonServiceException e1) {
        e1.printStackTrace();
        log.error("Promblem downloading crash file from S3 for " + s3FullFileName, e1);
    } catch (Exception e) {
        e.printStackTrace();
        log.error("Promblem downloading crash file from S3 for S3 for " + s3FullFileName, e);
    }

    log.info("Crash summary " + crashSummary);
    if (crashSummary != null && crashSummary.length() > 250) {
        crashSummary = crashSummary.substring(0, 249);
    }
    return crashSummary;
}

---

From source file:org.apache.usergrid.services.assets.data.AWSBinaryStore.java

License:Apache License

@Override
public InputStream read(UUID appId, Entity entity, long offset, long length) throws Exception {

    S3Object object = getS3Client().getObject(bucketName, AssetUtils.buildAssetKey(appId, entity));

    byte data[] = null;

    if (offset == 0 && length == FIVE_MB) {
        return object.getObjectContent();
    } else {//w  ww. j  a  v a  2  s  .  co  m
        object.getObjectContent().read(data, Ints.checkedCast(offset), Ints.checkedCast(length));
    }

    return new ByteArrayInputStream(data);
}

---

From source file:org.apache.zeppelin.notebook.repo.OldS3NotebookRepo.java

License:Apache License

private Note getNote(String key) throws IOException {
    S3Object s3object;
    try {/*w  w  w .  j  a v  a 2 s.com*/
        s3object = s3client.getObject(new GetObjectRequest(bucketName, key));
    } catch (AmazonClientException ace) {
        throw new IOException("Unable to retrieve object from S3: " + ace, ace);
    }

    try (InputStream ins = s3object.getObjectContent()) {
        String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
        return Note.fromJson(json);
    }
}

---

From source file:org.apache.zeppelin.notebook.repo.S3NotebookRepo.java

License:Apache License

@Override
public Note get(String noteId, String notePath, AuthenticationInfo subject) throws IOException {
    S3Object s3object;
    try {//from  w  w  w. j  a v a2 s.co m
        s3object = s3client.getObject(
                new GetObjectRequest(bucketName, rootFolder + "/" + buildNoteFileName(noteId, notePath)));
    } catch (AmazonClientException ace) {
        throw new IOException("Unable to retrieve object from S3: " + ace, ace);
    }
    try (InputStream ins = s3object.getObjectContent()) {
        String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
        return Note.fromJson(json);
    }
}

---

From source file:org.applicationMigrator.migrationclient.FileTransferClient.java

License:Apache License

private void writeObjectToFile(S3Object s3Object, String outputFilePath)
        throws IOException, InterruptedException {
    InputStream inputStream = null;
    inputStream = s3Object.getObjectContent();
    byte[] buf = new byte[1024];
    File outputFile = new File(outputFilePath);
    outputFile.setReadable(true, false);
    outputFile.setWritable(true, false);
    OutputStream outputStream = null;
    try {/*w  w w  .j  a v a2  s  .c  o m*/
        outputStream = new FileOutputStream(outputFile);
        int count;
        while ((count = inputStream.read(buf)) != -1) {
            if (Thread.interrupted()) {
                throw new InterruptedException();
            }
            outputStream.write(buf, 0, count);
        }
    } finally {
        if (outputStream != null)
            outputStream.close();
        if (inputStream != null)
            inputStream.close();
    }
}

---

From source file:org.boriken.s3fileuploader.S3SampleRefactored.java

License:Open Source License

public static void downloadFile(AmazonS3 s3, String bucketName, String key) throws IOException {
    /*//  ww w. jav  a2  s  .  co m
     * Download an object - When you download an object, you get all of
     * the object's metadata and a stream from which to read the contents.
     * It's important to read the contents of the stream as quickly as
     * possibly since the data is streamed directly from Amazon S3 and your
     * network connection will remain open until you read all the data or
     * close the input stream.
     *
     * GetObjectRequest also supports several other options, including
     * conditional downloading of objects based on modification times,
     * ETags, and selectively downloading a range of an object.
     */
    System.out.println("Downloading an object");
    S3Object object = s3.getObject(new GetObjectRequest(bucketName, key));
    System.out.println("Content-Type: " + object.getObjectMetadata().getContentType());
    displayTextInputStream(object.getObjectContent());
}

---

From source file:org.broadleafcommerce.vendor.amazon.s3.S3FileServiceProvider.java

License:Apache License

@Override
public File getResource(String name, FileApplicationType fileApplicationType) {
    final S3Configuration s3config = s3ConfigurationService.lookupS3Configuration();
    final String resourceName = buildResourceName(s3config, name);
    final File returnFile = blFileService.getLocalResource(resourceName);
    final String s3Uri = String.format("s3://%s/%s", s3config.getDefaultBucketName(), resourceName);

    OutputStream outputStream = null;
    InputStream inputStream = null;

    try {//from  w  ww  . j  a v  a  2  s.com
        final AmazonS3Client s3 = getAmazonS3Client(s3config);
        final S3Object object = s3.getObject(
                new GetObjectRequest(s3config.getDefaultBucketName(), buildResourceName(s3config, name)));

        if (LOG.isTraceEnabled()) {
            LOG.trace("retrieving " + s3Uri);
        }
        inputStream = object.getObjectContent();

        if (!returnFile.getParentFile().exists()) {
            if (!returnFile.getParentFile().mkdirs()) {
                // Other thread could have created - check one more time.
                if (!returnFile.getParentFile().exists()) {
                    throw new RuntimeException("Unable to create parent directories for file: " + name);
                }
            }
        }
        outputStream = new FileOutputStream(returnFile);
        int read = 0;
        byte[] bytes = new byte[1024];

        while ((read = inputStream.read(bytes)) != -1) {
            outputStream.write(bytes, 0, read);
        }
    } catch (IOException ioe) {
        throw new RuntimeException(String.format("Error writing %s to local file system at %s", s3Uri,
                returnFile.getAbsolutePath()), ioe);
    } catch (AmazonS3Exception s3Exception) {
        LOG.error(String.format("%s for %s; name = %s, resourceName = %s, returnFile = %s",
                s3Exception.getErrorCode(), s3Uri, name, resourceName, returnFile.getAbsolutePath()));

        if ("NoSuchKey".equals(s3Exception.getErrorCode())) {
            //return new File("this/path/should/not/exist/" + UUID.randomUUID());
            return null;
        } else {
            throw s3Exception;
        }
    } finally {
        if (inputStream != null) {
            try {
                inputStream.close();
            } catch (IOException e) {
                throw new RuntimeException("Error closing input stream while writing s3 file to file system",
                        e);
            }
        }
        if (outputStream != null) {
            try {
                outputStream.close();
            } catch (IOException e) {
                throw new RuntimeException("Error closing output stream while writing s3 file to file system",
                        e);
            }

        }
    }
    return returnFile;
}

---

From source file:org.caboclo.clients.AmazonClient.java

License:Open Source License

@Override
public void getFile(File file, String child) throws IOException {
    S3Object obj = s3.getObject(getBucketName(), child);

    BufferedInputStream bis = new BufferedInputStream(obj.getObjectContent());

    writeStreamToFile(bis, file);//  w ww .j  a va 2  s.  c  om
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• »