com.aliyun.odps.ship.upload.DshipUpload.java Source code

Java tutorial

Introduction

Here is the source code for com.aliyun.odps.ship.upload.DshipUpload.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package com.aliyun.odps.ship.upload;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import org.apache.commons.cli.ParseException;
import org.apache.commons.lang.StringUtils;

import com.aliyun.odps.ship.common.BlockInfo;
import com.aliyun.odps.ship.common.Constants;
import com.aliyun.odps.ship.common.DshipContext;
import com.aliyun.odps.ship.common.SessionStatus;
import com.aliyun.odps.ship.history.SessionHistory;
import com.aliyun.odps.ship.history.SessionHistoryManager;
import com.aliyun.odps.tunnel.TunnelException;
import com.aliyun.openservices.odps.console.ODPSConsoleException;

import jline.console.UserInterruptException;

/**
 * Created by lulu on 15-1-29.
 */
public class DshipUpload {

    private TunnelUploadSession tunnelUploadSession;
    private String sessionId;
    private SessionHistory sessionHistory;

    private boolean resume;
    private File uploadFile;
    private long totalUploadBytes;
    private ArrayList<BlockInfo> blockIndex = new ArrayList<BlockInfo>();
    private long blockSize = Constants.DEFAULT_BLOCK_SIZE * 1024 * 1024;
    private long startTime = System.currentTimeMillis();

    public DshipUpload() throws TunnelException, IOException, ParseException, ODPSConsoleException {
        resume = (DshipContext.INSTANCE.get(Constants.RESUME_UPLOAD_ID) != null);
        tunnelUploadSession = new TunnelUploadSession();
        sessionId = tunnelUploadSession.getUploadId();
        sessionHistory = SessionHistoryManager.createSessionHistory(sessionId);
        sessionHistory.saveContext();

        this.uploadFile = new File(DshipContext.INSTANCE.get(Constants.RESUME_PATH));
        this.totalUploadBytes = 0;
        if (DshipContext.INSTANCE.get(Constants.BLOCK_SIZE) != null) {
            blockSize = Long.valueOf(DshipContext.INSTANCE.get(Constants.BLOCK_SIZE)) * 1024 * 1024;
        }
        buildIndex(uploadFile);
    }

    public void upload() throws TunnelException, IOException, ParseException {

        System.err.println("Start upload:" + uploadFile.getPath());
        if (!resume) {
            System.err.println(
                    "Total bytes:" + totalUploadBytes + "\t Split input to " + blockIndex.size() + " blocks");
        } else {
            System.err.println("Resume " + blockIndex.size() + " blocks");
        }
        sessionHistory.log("start upload:" + uploadFile.getPath());

        String scan = DshipContext.INSTANCE.get(Constants.SCAN);
        if (isScan(scan)) {
            tunnelUploadSession.setScan(true);
            //only scan, don't really upload
            uploadBlock();
        }

        if (isUpload(scan)) {
            tunnelUploadSession.setScan(false);
            //really upload
            uploadBlock();
            List<BlockInfo> finishBlockList = sessionHistory.loadFinishBlockList();
            List<Long> finishBlockIdList = new ArrayList<Long>();
            for (BlockInfo block : finishBlockList) {
                finishBlockIdList.add(block.getBlockId());
            }
            tunnelUploadSession.complete(finishBlockIdList);
            long gap = (System.currentTimeMillis() - startTime) / 1000;
            if (gap > 0) {
                long avgSpeed = totalUploadBytes / gap / 1024;
                System.err.println("upload complete, average speed is " + avgSpeed + " KB/s");
            }
        }

        sessionHistory.log("upload complete:" + uploadFile.getPath());

        DshipContext.INSTANCE.put(Constants.STATUS, SessionStatus.success.toString());
        sessionHistory.saveContext();

        System.err.println("OK");
    }

    public String getTunnelSessionId() {
        return tunnelUploadSession.getUploadId();
    }

    private void uploadBlock() throws IOException, TunnelException, ParseException {
        int threads = Integer.valueOf(DshipContext.INSTANCE.get(Constants.THREADS));
        ExecutorService executors = Executors.newFixedThreadPool(threads);
        ArrayList<Callable<Long>> callList = new ArrayList<Callable<Long>>();
        for (BlockInfo block : blockIndex) {
            final BlockUploader uploader = new BlockUploader(block, tunnelUploadSession, sessionHistory);
            Callable<Long> call = new Callable<Long>() {
                @Override
                public Long call() throws Exception {
                    uploader.upload();
                    return 0L;
                }
            };

            callList.add(call);
        }

        try {
            List<Future<Long>> futures = executors.invokeAll(callList);
            ArrayList<String> failedBlock = new ArrayList<String>();
            for (int i = 0; i < futures.size(); ++i) {
                try {
                    futures.get(i).get();
                } catch (ExecutionException e) {
                    e.printStackTrace();
                    failedBlock.add(String.valueOf(i));
                }
            }
            if (!failedBlock.isEmpty()) {
                throw new TunnelException("Block ID:" + StringUtils.join(failedBlock, ",") + " Failed.");
            }
        } catch (InterruptedException e) {
            throw new UserInterruptException(e.getMessage());
        }
    }

    private void buildIndex(File file) throws IOException, TunnelException, ParseException {
        if (!resume) {
            BlockInfoBuilder blockIndexBuilder = new BlockInfoBuilder();
            blockIndexBuilder.setBlockSize(blockSize);

            blockIndex = blockIndexBuilder.buildBlockIndex(file);
            totalUploadBytes = blockIndexBuilder.getFileSize(file);
            sessionHistory.saveBlockIndex(blockIndex);
        } else {
            blockIndex = sessionHistory.loadBlockIndex();
        }
    }

    private boolean isScan(String scan) throws ParseException {
        if (scan.equalsIgnoreCase("true") || scan.equalsIgnoreCase("only")) {
            return true;
        } else if (scan.equalsIgnoreCase("false")) {
            return false;
        } else {
            throw new ParseException("Unrecognized command, '-scan='" + scan);
        }
    }

    private boolean isUpload(String scan) throws ParseException {
        if (scan.equalsIgnoreCase("true") || scan.equalsIgnoreCase("false")) {
            return true;
        } else if (scan.equalsIgnoreCase("only")) {
            return false;
        } else {
            throw new ParseException("Unrecognized command, '-scan='" + scan);
        }
    }

}