com.proofpoint.event.collector.combiner.S3CombineObjectMetadataStore.java Source code

Introduction

Here is the source code for com.proofpoint.event.collector.combiner.S3CombineObjectMetadataStore.java

Source

/*
 * Copyright 2011-2014 Proofpoint, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.proofpoint.event.collector.combiner;

import com.amazonaws.AmazonClientException;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.google.common.base.Charsets;
import com.proofpoint.event.collector.EventPartition;
import com.proofpoint.event.collector.ServerConfig;
import com.proofpoint.json.JsonCodec;
import com.proofpoint.log.Logger;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.codec.digest.DigestUtils;

import javax.inject.Inject;
import javax.ws.rs.core.MediaType;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.proofpoint.event.collector.combiner.S3StorageHelper.getS3Bucket;
import static com.proofpoint.event.collector.combiner.S3StorageHelper.getS3ObjectKey;

public class S3CombineObjectMetadataStore implements CombineObjectMetadataStore {
    private static final Logger log = Logger.get(S3CombineObjectMetadataStore.class);
    private final JsonCodec<CombinedGroup> jsonCodec = JsonCodec.jsonCodec(CombinedGroup.class);
    private final AmazonS3 s3Service;
    private final URI storageArea;

    @Inject
    public S3CombineObjectMetadataStore(ServerConfig config, AmazonS3 s3Service) {
        this(URI.create(checkNotNull(config, "config is null").getS3MetadataLocation()), s3Service);
    }

    public S3CombineObjectMetadataStore(URI storageArea, AmazonS3 s3Service) {
        this.storageArea = checkNotNull(storageArea, "storageArea is null");
        this.s3Service = checkNotNull(s3Service, "s3Service is null");
    }

    @Override
    public CombinedGroup getCombinedGroupManifest(EventPartition eventPartition, String sizeName) {
        return readMetadataFile(eventPartition, sizeName);
    }

    @Override
    public boolean replaceCombinedGroupManifest(EventPartition eventPartition, String sizeName,
            CombinedGroup currentGroup, CombinedGroup newGroup) {
        checkNotNull(currentGroup, "currentGroup is null");
        checkNotNull(newGroup, "newGroup is null");
        checkArgument(currentGroup.getLocationPrefix().equals(newGroup.getLocationPrefix()),
                "newGroup location is different from currentGroup location");

        CombinedGroup persistentGroup = readMetadataFile(eventPartition, sizeName);
        if (persistentGroup != null) {
            if (persistentGroup.getVersion() != currentGroup.getVersion()) {
                return false;
            }
        } else if (currentGroup.getVersion() != 0) {
            return false;
        }

        return writeMetadataFile(eventPartition, newGroup, sizeName);
    }

    private boolean writeMetadataFile(EventPartition eventPartition, CombinedGroup combinedGroup, String sizeName) {
        byte[] json = jsonCodec.toJson(combinedGroup).getBytes(Charsets.UTF_8);
        URI metadataFile = toMetadataLocation(eventPartition, sizeName);
        try {
            ObjectMetadata metadata = new ObjectMetadata();
            metadata.setContentLength(json.length);
            metadata.setContentMD5(Base64.encodeBase64String(DigestUtils.md5(json)));
            metadata.setContentType(MediaType.APPLICATION_JSON);
            InputStream input = new ByteArrayInputStream(json);
            s3Service.putObject(getS3Bucket(metadataFile), getS3ObjectKey(metadataFile), input, metadata);
            return true;
        } catch (AmazonClientException e) {
            log.warn(e, "error writing metadata file: %s", metadataFile);
            return false;
        }
    }

    private URI toMetadataLocation(EventPartition eventPartition, String sizeName) {
        return storageArea.resolve(eventPartition.getEventType() + "/" + eventPartition.getMajorTimeBucket() + "/"
                + eventPartition.getMinorTimeBucket() + "." + sizeName + ".metadata");
    }

    private CombinedGroup readMetadataFile(EventPartition eventPartition, String sizeName) {
        URI metadataFile = toMetadataLocation(eventPartition, sizeName);
        String json;
        try {
            json = new S3InputSupplier(s3Service, metadataFile).asCharSource(Charsets.UTF_8).read();
        } catch (IOException e) {
            if (e.getCause() instanceof AmazonS3Exception) {
                if ("NoSuchKey".equals(((AmazonS3Exception) e.getCause()).getErrorCode())) {
                    return null;
                }
            }
            throw new RuntimeException(
                    "Could not load metadata at " + metadataFile + " file for " + eventPartition + " " + sizeName);
        }
        try {
            return jsonCodec.fromJson(json);
        } catch (IllegalArgumentException e) {
            throw new RuntimeException(
                    "Metadata at " + metadataFile + " file for " + eventPartition + " " + sizeName + " is corrupt");
        }
    }
}