ch.cyberduck.core.openstack.SwiftLargeObjectUploadFeature.java Source code

Java tutorial

Introduction

Here is the source code for ch.cyberduck.core.openstack.SwiftLargeObjectUploadFeature.java

Source

package ch.cyberduck.core.openstack;

/*
 * Copyright (c) 2013 David Kocher. All rights reserved.
 * http://cyberduck.ch/
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * Bug fixes, suggestions and comments should be sent to:
 * feedback@cyberduck.ch
 */

import ch.cyberduck.core.ConnectionCallback;
import ch.cyberduck.core.DefaultIOExceptionMappingService;
import ch.cyberduck.core.DisabledListProgressListener;
import ch.cyberduck.core.Local;
import ch.cyberduck.core.Path;
import ch.cyberduck.core.PathContainerService;
import ch.cyberduck.core.exception.BackgroundException;
import ch.cyberduck.core.exception.ConnectionCanceledException;
import ch.cyberduck.core.features.Upload;
import ch.cyberduck.core.features.Write;
import ch.cyberduck.core.http.HttpUploadFeature;
import ch.cyberduck.core.io.BandwidthThrottle;
import ch.cyberduck.core.io.HashAlgorithm;
import ch.cyberduck.core.io.StreamCopier;
import ch.cyberduck.core.io.StreamListener;
import ch.cyberduck.core.io.StreamProgress;
import ch.cyberduck.core.threading.BackgroundExceptionCallable;
import ch.cyberduck.core.threading.DefaultRetryCallable;
import ch.cyberduck.core.threading.DefaultThreadPool;
import ch.cyberduck.core.threading.ThreadPool;
import ch.cyberduck.core.transfer.TransferStatus;
import ch.cyberduck.core.worker.DefaultExceptionMappingService;

import org.apache.commons.io.input.BoundedInputStream;
import org.apache.log4j.Logger;

import java.io.IOException;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;

import ch.iterate.openstack.swift.exception.GenericException;
import ch.iterate.openstack.swift.model.StorageObject;

public class SwiftLargeObjectUploadFeature extends HttpUploadFeature<StorageObject, MessageDigest> {
    private static final Logger log = Logger.getLogger(SwiftLargeObjectUploadFeature.class);

    private final SwiftSession session;

    private final PathContainerService containerService = new PathContainerService();

    private final SwiftSegmentService segmentService;
    private final SwiftObjectListService listService;
    private final SwiftRegionService regionService;

    private final Long segmentSize;
    private final Integer concurrency;

    private Write<StorageObject> writer;

    public SwiftLargeObjectUploadFeature(final SwiftSession session, final SwiftRegionService regionService,
            final Write<StorageObject> writer, final Long segmentSize, final Integer concurrency) {
        this(session, regionService, new SwiftObjectListService(session, regionService),
                new SwiftSegmentService(session, regionService), writer, segmentSize, concurrency);
    }

    public SwiftLargeObjectUploadFeature(final SwiftSession session, final SwiftRegionService regionService,
            final SwiftObjectListService listService, final SwiftSegmentService segmentService,
            final Write<StorageObject> writer, final Long segmentSize, final Integer concurrency) {
        super(writer);
        this.session = session;
        this.regionService = regionService;
        this.writer = writer;
        this.segmentSize = segmentSize;
        this.segmentService = segmentService;
        this.listService = listService;
        this.concurrency = concurrency;
    }

    @Override
    public StorageObject upload(final Path file, final Local local, final BandwidthThrottle throttle,
            final StreamListener listener, final TransferStatus status, final ConnectionCallback callback)
            throws BackgroundException {
        final DefaultThreadPool pool = new DefaultThreadPool("multipart", concurrency);
        final List<Path> existingSegments = new ArrayList<Path>();
        if (status.isAppend() || status.isRetry()) {
            // Get a lexicographically ordered list of the existing file segments
            existingSegments
                    .addAll(listService
                            .list(segmentService.getSegmentsDirectory(file,
                                    status.getOffset() + status.getLength()), new DisabledListProgressListener())
                            .toList());
        }
        // Get the results of the uploads in the order they were submitted
        // this is important for building the manifest, and is not a problem in terms of performance
        // because we should only continue when all segments have uploaded successfully
        final List<StorageObject> completed = new ArrayList<StorageObject>();
        // Submit file segments for concurrent upload
        final List<Future<StorageObject>> segments = new ArrayList<Future<StorageObject>>();
        long remaining = status.getLength();
        long offset = 0;
        for (int segmentNumber = 1; remaining > 0; segmentNumber++) {
            final Long length = Math.min(segmentSize, remaining);
            // Segment name with left padded segment number
            final Path segment = segmentService.getSegment(file, status.getOffset() + status.getLength(),
                    segmentNumber);
            if (existingSegments.contains(segment)) {
                final Path existingSegment = existingSegments.get(existingSegments.indexOf(segment));
                if (log.isDebugEnabled()) {
                    log.debug(String.format("Skip segment %s", existingSegment));
                }
                final StorageObject stored = new StorageObject(containerService.getKey(segment));
                if (HashAlgorithm.md5.equals(existingSegment.attributes().getChecksum().algorithm)) {
                    stored.setMd5sum(existingSegment.attributes().getChecksum().hash);
                }
                stored.setSize(existingSegment.attributes().getSize());
                offset += existingSegment.attributes().getSize();
                completed.add(stored);
            } else {
                // Submit to queue
                segments.add(
                        this.submit(pool, segment, local, throttle, listener, status, offset, length, callback));
                if (log.isDebugEnabled()) {
                    log.debug(String.format("Segment %s submitted with size %d and offset %d", segment, length,
                            offset));
                }
                remaining -= length;
                offset += length;
            }
        }
        try {
            for (Future<StorageObject> futureSegment : segments) {
                completed.add(futureSegment.get());
            }
        } catch (InterruptedException e) {
            log.error("Part upload failed with interrupt failure");
            status.setCanceled();
            throw new ConnectionCanceledException(e);
        } catch (ExecutionException e) {
            log.warn(String.format("Part upload failed with execution failure %s", e.getMessage()));
            if (e.getCause() instanceof BackgroundException) {
                throw (BackgroundException) e.getCause();
            }
            throw new DefaultExceptionMappingService().map(e.getCause());
        } finally {
            pool.shutdown(false);
        }
        // Mark parent status as complete
        status.setComplete();
        if (log.isInfoEnabled()) {
            log.info(String.format("Finished large file upload %s with %d parts", file, completed.size()));
        }
        // Create and upload the large object manifest. It is best to upload all the segments first and
        // then create or update the manifest.
        try {
            // Static Large Object.
            final String manifest = segmentService.manifest(containerService.getContainer(file).getName(),
                    completed);
            if (log.isDebugEnabled()) {
                log.debug(String.format("Creating SLO manifest %s for %s", manifest, file));
            }
            final StorageObject stored = new StorageObject(containerService.getKey(file));
            final String checksum = session.getClient().createSLOManifestObject(
                    regionService.lookup(containerService.getContainer(file)),
                    containerService.getContainer(file).getName(), status.getMime(), containerService.getKey(file),
                    manifest, Collections.emptyMap());
            // The value of the Content-Length header is the total size of all segment objects, and the value of the ETag header is calculated by taking
            // the ETag value of each segment, concatenating them together, and then returning the MD5 checksum of the result.
            stored.setMd5sum(checksum);
            return stored;
        } catch (GenericException e) {
            throw new SwiftExceptionMappingService().map("Upload {0} failed", e);
        } catch (IOException e) {
            throw new DefaultIOExceptionMappingService().map("Upload {0} failed", e, file);
        }
    }

    private Future<StorageObject> submit(final ThreadPool pool, final Path segment, final Local local,
            final BandwidthThrottle throttle, final StreamListener listener, final TransferStatus overall,
            final Long offset, final Long length, final ConnectionCallback callback) {
        return pool
                .execute(new DefaultRetryCallable<StorageObject>(new BackgroundExceptionCallable<StorageObject>() {
                    @Override
                    public StorageObject call() throws BackgroundException {
                        if (overall.isCanceled()) {
                            throw new ConnectionCanceledException();
                        }
                        final TransferStatus status = new TransferStatus().length(length).skip(offset);
                        status.setHeader(overall.getHeader());
                        status.setNonces(overall.getNonces());
                        status.setChecksum(writer.checksum(segment)
                                .compute(StreamCopier.skip(
                                        new BoundedInputStream(local.getInputStream(), offset + length), offset),
                                        status));
                        status.setSegment(true);
                        return SwiftLargeObjectUploadFeature.super.upload(segment, local, throttle, listener,
                                status, overall, new StreamProgress() {
                                    @Override
                                    public void progress(final long bytes) {
                                        status.progress(bytes);
                                        // Discard sent bytes in overall progress if there is an error reply for segment.
                                        overall.progress(bytes);
                                    }

                                    @Override
                                    public void setComplete() {
                                        status.setComplete();
                                    }
                                }, callback);
                    }
                }, overall));
    }

    @Override
    public Upload<StorageObject> withWriter(final Write<StorageObject> writer) {
        this.writer = writer;
        return super.withWriter(writer);
    }
}