org.apache.hive.hcatalog.templeton.HiveDelegator.java Source code

Introduction

Here is the source code for org.apache.hive.hcatalog.templeton.HiveDelegator.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.hive.hcatalog.templeton;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;

import org.apache.commons.exec.ExecuteException;
import org.apache.hadoop.fs.Path;
import org.apache.hive.hcatalog.templeton.tool.JobSubmissionConstants;
import org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob;
import org.apache.hive.hcatalog.templeton.tool.TempletonUtils;

/**
 * Submit a Hive job.
 *
 * This is the backend of the hive web service.
 */
public class HiveDelegator extends LauncherDelegator {

    public HiveDelegator(AppConfig appConf) {
        super(appConf);
    }

    public EnqueueBean run(String user, Map<String, Object> userArgs, String execute, String srcFile,
            List<String> defines, List<String> hiveArgs, String otherFiles, String statusdir, String callback,
            String completedUrl, boolean enablelog, Boolean enableJobReconnect)
            throws NotAuthorizedException, BadParam, BusyException, QueueException, ExecuteException, IOException,
            InterruptedException, TooManyRequestsException {
        runAs = user;
        List<String> args = makeArgs(execute, srcFile, defines, hiveArgs, otherFiles, statusdir, completedUrl,
                enablelog, enableJobReconnect);

        return enqueueController(user, userArgs, callback, args);
    }

    private List<String> makeArgs(String execute, String srcFile, List<String> defines, List<String> hiveArgs,
            String otherFiles, String statusdir, String completedUrl, boolean enablelog, Boolean enableJobReconnect)
            throws BadParam, IOException, InterruptedException {
        ArrayList<String> args = new ArrayList<String>();
        try {
            args.addAll(makeBasicArgs(execute, srcFile, otherFiles, statusdir, completedUrl, enablelog,
                    enableJobReconnect));
            args.add("--");
            addHiveMetaStoreTokenArg();

            args.add(appConf.hivePath());

            args.add("--service");
            args.add("cli");

            //the token file location as initial hiveconf arg
            args.add("--hiveconf");
            args.add(TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER);

            //this is needed specifcally for Hive on Tez (in addition to
            //JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER)
            args.add("--hiveconf");
            args.add(JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ);

            //add mapreduce job tag placeholder
            args.add("--hiveconf");
            args.add(TempletonControllerJob.MAPREDUCE_JOB_TAGS_ARG_PLACEHOLDER);

            for (String prop : appConf.hiveProps()) {
                args.add("--hiveconf");
                args.add(prop);
            }
            for (String prop : defines) {
                args.add("--hiveconf");
                args.add(prop);
            }
            for (String hiveArg : hiveArgs) {
                args.add(hiveArg);
            }
            if (TempletonUtils.isset(execute)) {
                args.add("-e");
                args.add(execute);
            } else if (TempletonUtils.isset(srcFile)) {
                args.add("-f");
                args.add(TempletonUtils.hadoopFsPath(srcFile, appConf, runAs).getName());
            }
        } catch (FileNotFoundException e) {
            throw new BadParam(e.getMessage());
        } catch (URISyntaxException e) {
            throw new BadParam(e.getMessage());
        }

        return args;
    }

    private List<String> makeBasicArgs(String execute, String srcFile, String otherFiles, String statusdir,
            String completedUrl, boolean enablelog, Boolean enableJobReconnect)
            throws URISyntaxException, IOException, InterruptedException {
        ArrayList<String> args = new ArrayList<String>();

        ArrayList<String> allFiles = new ArrayList<String>();
        if (TempletonUtils.isset(srcFile)) {
            allFiles.add(TempletonUtils.hadoopFsFilename(srcFile, appConf, runAs));
        }

        if (TempletonUtils.isset(otherFiles)) {
            String[] ofs = TempletonUtils.hadoopFsListAsArray(otherFiles, appConf, runAs);
            allFiles.addAll(Arrays.asList(ofs));
        }

        args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles, enablelog, enableJobReconnect,
                JobType.HIVE));

        if (appConf.hiveArchive() != null && !appConf.hiveArchive().equals("")) {
            args.add(ARCHIVES);
            args.add(appConf.hiveArchive());
        }

        //ship additional artifacts, for example for Tez
        String extras = appConf.get(AppConfig.HIVE_EXTRA_FILES);
        if (extras != null && extras.length() > 0) {
            boolean foundFiles = false;
            for (int i = 0; i < args.size(); i++) {
                if (FILES.equals(args.get(i))) {
                    String value = args.get(i + 1);
                    args.set(i + 1, value + "," + extras);
                    foundFiles = true;
                }
            }
            if (!foundFiles) {
                args.add(FILES);
                args.add(extras);
            }
            String[] extraFiles = appConf.getStrings(AppConfig.HIVE_EXTRA_FILES);
            StringBuilder extraFileNames = new StringBuilder();
            //now tell LaunchMapper which files it should add to HADOOP_CLASSPATH
            for (String file : extraFiles) {
                Path p = new Path(file);
                extraFileNames.append(p.getName()).append(",");
            }
            addDef(args, JobSubmissionConstants.HADOOP_CLASSPATH_EXTRAS, extraFileNames.toString());
        }
        return args;
    }
}