io.dockstore.webservice.core.tooltester.ToolTesterS3Client.java Source code

Java tutorial

Introduction

Here is the source code for io.dockstore.webservice.core.tooltester.ToolTesterS3Client.java

Source

/*
 *
 *  *    Copyright 2019 OICR
 *  *
 *  *    Licensed under the Apache License, Version 2.0 (the "License");
 *  *    you may not use this file except in compliance with the License.
 *  *    You may obtain a copy of the License at
 *  *
 *  *        http://www.apache.org/licenses/LICENSE-2.0
 *  *
 *  *    Unless required by applicable law or agreed to in writing, software
 *  *    distributed under the License is distributed on an "AS IS" BASIS,
 *  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  *    See the License for the specific language governing permissions and
 *  *    limitations under the License.
 *
 */

package io.dockstore.webservice.core.tooltester;

import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import org.apache.commons.io.IOUtils;

/**
 * @author gluu
 * @since 24/04/19
 */
public class ToolTesterS3Client {
    private static final int MAX_TOOL_ID_STRING_SEGMENTS = 5;
    private static final int TOOL_ID_REPOSITORY_INDEX = 3;
    private static final int TOOL_ID_TOOLNAME_INDEX = 4;
    private String bucketName;
    private AmazonS3 s3;

    public ToolTesterS3Client(String bucketName) {
        this.bucketName = bucketName;
        this.s3 = AmazonS3ClientBuilder.standard().build();
    }

    /**
     * This essentially generates the s3 key of the log that is already be stored on S3.
     * Different from the ToolTester function of the same name, as that takes different parameters.
     *
     * @param toolId       The GA4GH Tool ID
     * @param versionName  The GA4GH ToolVersion name
     * @param testFilePath The file that was tested (Dockerfile, test.json, etc)
     * @param runner       The runner used to test (cwltool, cromwell, etc)
     * @param filename     The log name time in milliseconds since epoch
     * @return S3 key (file path)
     * @throws UnsupportedEncodingException Could not endpoint string
     */
    private static String generateKey(String toolId, String versionName, String testFilePath, String runner,
            String filename) throws UnsupportedEncodingException {
        List<String> pathList = new ArrayList<>();
        pathList.add(convertToolIdToPartialKey(toolId));
        pathList.add(URLEncoder.encode(versionName, StandardCharsets.UTF_8.name()));
        pathList.add(URLEncoder.encode(testFilePath, StandardCharsets.UTF_8.name()));
        pathList.add(runner);
        pathList.add(filename);
        return String.join("/", pathList);
    }

    static ToolTesterLog convertUserMetadataToToolTesterLog(Map<String, String> userMetadata, String filename) {
        String toolId = userMetadata.get(ObjectMetadataEnum.TOOL_ID.toString());
        String toolVersionName = userMetadata.get(ObjectMetadataEnum.VERSION_NAME.toString());
        String testFilename = userMetadata.get(ObjectMetadataEnum.TEST_FILE_PATH.toString());
        String runner = userMetadata.get(ObjectMetadataEnum.RUNNER.toString());
        ToolTesterLogType logType = ToolTesterLogType.FULL;
        return new ToolTesterLog(toolId, toolVersionName, testFilename, runner, logType, filename);
    }

    /**
     * Converts the toolId into a key for s3 storage.  Used by both webservice and tooltester
     * Workflows will be in a "workflow" directory whereas tools will be in a "tool" directory
     * repository and optional toolname or workflowname must be encoded or else looking for logs of a specific tool without toolname (quay.io/dockstore/hello_world)
     * will return logs for the other ones with toolnames (quay.io/dockstore/hello_world/thing)
     * TODO: Somehow reuse this between repos
     *
     * @param toolId TRS tool ID
     * @return The key for s3
     */
    static String convertToolIdToPartialKey(String toolId) throws UnsupportedEncodingException {
        String partialKey = toolId;
        if (partialKey.startsWith("#workflow")) {
            partialKey = partialKey.replaceFirst("#workflow", "workflow");
        } else {
            partialKey = "tool/" + partialKey;
        }
        String[] split = partialKey.split("/");
        if (split.length == MAX_TOOL_ID_STRING_SEGMENTS) {
            split[TOOL_ID_REPOSITORY_INDEX] = URLEncoder.encode(
                    split[TOOL_ID_REPOSITORY_INDEX] + "/" + split[TOOL_ID_TOOLNAME_INDEX],
                    StandardCharsets.UTF_8.name());
            String[] encodedToolIdArray = Arrays.copyOf(split, split.length - 1);
            return String.join("/", encodedToolIdArray);
        } else {
            return partialKey;
        }
    }

    public String getToolTesterLog(String toolId, String versionName, String testFilePath, String runner,
            String filename) throws IOException {
        String key = generateKey(toolId, versionName, testFilePath, runner, filename);
        S3Object s3Object = s3.getObject(bucketName, key);
        InputStream objectContent = s3Object.getObjectContent();
        return IOUtils.toString(objectContent, StandardCharsets.UTF_8);

    }

    public List<ToolTesterLog> getToolTesterLogs(String toolId, String toolVersionName)
            throws UnsupportedEncodingException {
        String key = convertToolIdToPartialKey(toolId) + "/"
                + URLEncoder.encode(toolVersionName, StandardCharsets.UTF_8.name());
        ObjectListing listing = s3.listObjects(bucketName, key);
        List<S3ObjectSummary> summaries = listing.getObjectSummaries();
        while (listing.isTruncated()) {
            listing = s3.listNextBatchOfObjects(listing);
            summaries.addAll(listing.getObjectSummaries());
        }
        return convertObjectListingToTooltesterLogs(listing);
    }

    private List<ToolTesterLog> convertObjectListingToTooltesterLogs(ObjectListing firstListing) {
        ObjectListing listing = firstListing;
        List<S3ObjectSummary> summaries = listing.getObjectSummaries();
        while (listing.isTruncated()) {
            listing = s3.listNextBatchOfObjects(listing);
            summaries.addAll(listing.getObjectSummaries());
        }
        return summaries.stream().map(summary -> {
            ObjectMetadata objectMetadata = s3.getObjectMetadata(bucketName, summary.getKey());
            Map<String, String> userMetadata = objectMetadata.getUserMetadata();
            String filename = getFilenameFromSummary(summary);
            return convertUserMetadataToToolTesterLog(userMetadata, filename);
        }).collect(Collectors.toList());
    }

    private String getFilenameFromSummary(S3ObjectSummary summary) {
        String key = summary.getKey();
        return summary.getKey().substring(key.lastIndexOf("/") + 1);
    }
}