org.aliuge.crawler.extractor.selector.action.file.DownLoadFileAction.java Source code

Introduction

Here is the source code for org.aliuge.crawler.extractor.selector.action.file.DownLoadFileAction.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.aliuge.crawler.extractor.selector.action.file;

import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Date;
import java.util.List;
import java.util.Map;

import org.aliuge.crawler.exception.DownLoadException;
import org.aliuge.crawler.extractor.selector.action.FileSelectAction;
import org.aliuge.crawler.util.DateTimeUtil;
import org.aliuge.crawler.util.MD5Utils;
import org.aliuge.crawler.util.MultiThreadDownload;
import org.apache.commons.lang3.StringUtils;

import com.google.common.collect.Lists;

/**
 * @author 
 * @date 2015-8-3
 * @desc ??????
 */
public class DownLoadFileAction extends FileSelectAction {
    /**
     * 
     */
    File dir = null;
    /**
     * ?
     */
    List<String> dynamicPath = null;
    /**
     * ?Urlmd5??false???
     */
    boolean md5File = true;
    /**
     * ??
     */
    boolean asynchronous = true;
    /**
     * ??
     */
    long blockSize = 1024 * 1024L;

    /**
     * 
     * @param dir
     * @param md5File
     */
    public DownLoadFileAction(String dir, boolean md5File) {
        if (!dir.contains("{")) {
            this.dir = new File(dir);
            if (!this.dir.exists()) {
                this.dir.mkdir();
            }
        } else {
            this.dynamicPath = Lists.newArrayList(StringUtils.substringsBetween(dir, "{", "}"));
            String ss[] = StringUtils.split(dir, "/");
            for (String s : ss) {
                if (!s.contains("{")) {
                    this.dynamicPath.add(0, s);
                }
            }
        }
        this.md5File = md5File;
    }

    /**
     * 
     * @param dir
     * @param md5File
     * @param asynchronous
     */
    public DownLoadFileAction(String dir, boolean md5File, boolean asynchronous) {
        super();
        if (!dir.contains("{")) {
            this.dir = new File(dir);
            if (!this.dir.exists()) {
                this.dir.mkdir();
            }
        } else {
            this.dynamicPath = Lists.newArrayList(StringUtils.substringsBetween(dir, "{", "}"));
            String ss[] = StringUtils.split(dir, "/");
            for (String s : ss) {
                if (!s.contains("{")) {
                    this.dynamicPath.add(0, s);
                }
            }
        }
        this.md5File = md5File;
        this.asynchronous = asynchronous;
    }

    /**
     * 
     */
    @Override
    public String doAction(Map<String, Object> result, String remoteFile) throws DownLoadException {
        // 
        String path = getDownDir(result);
        URL url;
        try {
            url = new URL(remoteFile);
        } catch (MalformedURLException e) {
            e.printStackTrace();
            throw new DownLoadException("" + e.getMessage());
        }
        String fileName = "";
        if (md5File) {
            try {
                fileName = MD5Utils.createMD5(remoteFile);
                fileName = fileName + "." + StringUtils.substringAfterLast(url.getPath(), ".");
            } catch (Exception e) {
            }
        } else {
            fileName = StringUtils.substringAfterLast(url.getPath(), "/");
        }
        MultiThreadDownload download = new MultiThreadDownload(1024 * 1024L);
        if (asynchronous) {
            download.downFile(url, path, fileName, false);
        } else {
            download.downLoad(url, path, fileName);
        }
        // 
        return StringUtils.replace(path + fileName, "\\", "/");
    }

    /**
     * 
     * @param map
     * @return
     */
    public String getDownDir(Map<String, Object> map) {
        if (null != map) {
            if (null != dir) {
                return dir.getPath();
            } else if (null != dynamicPath) {
                StringBuilder sb = new StringBuilder();
                for (String p : dynamicPath) {
                    Object o = map.get(p);
                    if (null != o && o instanceof Date) {
                        sb.append(DateTimeUtil.getYearOfDate((Date) o)).append(File.separator);
                    } else {
                        sb.append(String.valueOf(map.get(p))).append(File.separator);
                    }
                }
                return sb.toString();
            }
        }
        return "";
    }

    public static void main(String[] args) {
        //      System.out.println(StringUtils.substringAfterLast("sdfasd.asdfaf.gif", "."));
        DownLoadFileAction downLoadFileAction = new DownLoadFileAction("d:/multidown", true);
        try {
            downLoadFileAction.doAction(null,
                    "http://zhangmenshiting.baidu.com/data2/music/65517089/307842151200128.mp3?xcode=53102624c6c63d206dbeaf3b8ae12d9080af3c8af038c7a6");
        } catch (DownLoadException e) {
            e.printStackTrace();
        }
    }
}