com.aliyun.odps.mapred.conf.SessionState.java Source code

Java tutorial

Introduction

Here is the source code for com.aliyun.odps.mapred.conf.SessionState.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package com.aliyun.odps.mapred.conf;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.security.AccessControlException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;

import org.apache.commons.io.FileUtils;

import com.alibaba.fastjson.JSON;
import com.aliyun.odps.Odps;
import com.aliyun.odps.OdpsHook;
import com.aliyun.odps.OdpsHooks;
import com.aliyun.odps.account.Account;
import com.aliyun.odps.account.Account.AccountProvider;
import com.aliyun.odps.account.AliyunAccount;
import com.aliyun.odps.utils.StringUtils;

/**
 * SessionStateOdps MapReduce?
 *
 * SessionState??
 * <p>
 * <li>{@link com.aliyun.odps.Odps}</li>
 * <li>{@link com.aliyun.odps.mapred.conf.JobConf}</li>
 * <li>local run </li>
 * </p>
 *
 * ????local run
 *
 * <pre>
 * SessionState ss = SessionState.get();
 * ss.setLocalRun(true);
 * </pre>
 */
public class SessionState {

    private static ThreadLocal<SessionState> tss = new InheritableThreadLocal<SessionState>() {
        @Override
        public SessionState initialValue() {
            return new SessionState();
        }

        @Override
        protected SessionState childValue(SessionState parentValue) {
            return new SessionState(parentValue);
        }
    };

    private Odps odps;
    private JobConf defaultJob;
    private boolean isLocalRun;
    private boolean isCostMode;
    private Map<String, String> aliases;
    // if use internal cli
    private boolean internalCli = false;
    private String commandText = "";

    private SessionState() {
        try {
            aliases = new HashMap<String, String>();
            defaultJob = new JobConf(false);
            String conf = System.getProperties().getProperty(MR_JOB_CONF);
            if (conf != null && !conf.isEmpty()) {
                try {
                    defaultJob.addResource(new FileInputStream(new File(conf)));
                } catch (FileNotFoundException e) {
                    System.err.print("conf file " + conf + " not found, ignored!");
                }
            }
            parseOldCli(defaultJob); // Try to be compatible with old console
        } catch (AccessControlException e) {
            // just ignore it for it maybe called by user code indirectly through JobConf constructors 
        }
    }

    public SessionState(SessionState ss) {
        if (ss.odps != null) {
            odps = new Odps(ss.odps);
        }
        if (ss.defaultJob != null) {
            defaultJob = new JobConf(ss.defaultJob);
        }
        isLocalRun = ss.isLocalRun;
        aliases = new HashMap<String, String>();
        if (ss.aliases != null) {
            aliases.putAll(ss.aliases);
        }
    }

    // Copied from old OdpsConf.
    private final static String OLD_PROJNAME_KEY = "odps.project.name";
    private final static String OLD_ENDPOINT_KEY = "odps.end.point";
    private final static String OLD_ACCESSID_KEY = "odps.access.id";
    private final static String OLD_ACCESSKEY_KEY = "odps.access.key";
    private final static String OLD_RUNMODE_KEY = "odps.runner.mode";
    private final static String OLD_RESOURCE_KEY = "odps.cache.resources";
    private final static String OLD_CP_RESOURCE_KEY = "odps.classpath.resources";
    private final static String OLD_CONTEXT_FILE = "odps.exec.context.file";

    private final static String OLD_ACCOUNT_PROVIDER_KEY = "odps.account.provider";
    private final static String OLD_TAOBAO_TOKEN_KEY = "odps.taobao.token";
    private final static String OLD_TAOBAO_ALGORITHM_KEY = "odps.taobao.algorithm";

    // Local MR
    private final static String LOCAL_TEMP_DIR = "odps.mapred.local.temp.dir";
    private final static String LOCAL_TEMP_RETAIN = "odps.mapred.local.temp.retain";
    private final static String LOCAL_RECORD_LIMIT = "odps.mapred.local.record.download.limit";
    public static final String LOCAL_DOWNLOAD_MODE = "odps.mapred.local.download.mode"; //always|auto|never; default auto
    //Local security
    public static final String LOCAL_SECURITY_ENABLE = "odps.local.security.enable";
    public static final String LOCAL_SECURITY_JNI_ENABLE = "odps.local.security.jni.enable";
    public static final String LOCAL_USER_DEFINE_POLICY = "odps.local.user.define.policy";

    public static final String COST = "cost";
    public static final String ODPS_JOB_COST_ESTIMATE = "odps.task.cost.estimate";

    private final static String LOCAL_INPUT_COLUMN_SEPERATOR = "odps.mapred.local.input.column.seperator";
    private final static String LOCAL_OUTPUT_COLUMN_SEPERATOR = "odps.mapred.local.output.column.seperator";

    private static final String MR_JOB_CONF = "odps.mr.job.conf";

    /**
     * From old Console load context file , include settings & aliases
     *
     * @return
     */
    @SuppressWarnings({ "unchecked", "rawtypes" })
    private void loadContextFile(Properties prop) throws IOException {
        String fileName = prop.getProperty(OLD_CONTEXT_FILE);
        if (fileName == null) {
            return;
        }

        String jsonStr = FileUtils.readFileToString(new File(fileName));
        Map context = JSON.parseObject(jsonStr, Map.class);
        if (context == null) {
            return;
        }
        // client set ignore certs to true when request from internal-cli
        internalCli = true;
        getOdps().getRestClient().setIgnoreCerts(true);
        if (context.containsKey("settings")) {
            Map<String, String> settings = (Map<String, String>) context.get("settings");
            for (Entry<String, String> setting : settings.entrySet()) {
                defaultJob.set(setting.getKey(), setting.getValue());
            }
        }

        if (context.containsKey("aliases")) {
            aliases = (Map<String, String>) context.get("aliases");
        }

        if (context.containsKey("context")) {
            // extract info from execution context
            Map<String, Object> ctx = (Map<String, Object>) context.get("context");
            defaultJob.setInstancePriority((Integer) ctx.get("priority"));
            OdpsHooks.clearRegisteredHooks();
            String hookString = (String) ctx.get("odpsHooks");
            if (!StringUtils.isNullOrEmpty(hookString)) {
                try {
                    String[] hooks = hookString.split(",");
                    List<Class<? extends OdpsHook>> hookList = new ArrayList<Class<? extends OdpsHook>>();
                    for (String hook : hooks) {
                        hookList.add((Class<? extends OdpsHook>) Class.forName(hook));
                    }
                    OdpsHooks.registerHooks(hookList);

                } catch (ClassNotFoundException e) {
                    throw new IOException(e.getMessage(), e);
                }
            }

            String runningCluster = (String) ctx.get("runningCluster");
            if (!StringUtils.isNullOrEmpty(runningCluster)) {
                odps.instances().setDefaultRunningCluster(runningCluster);
            }

            if (ctx.get("logViewHost") != null) {
                String logViewHost = (String) ctx.get("logViewHost");
                odps.setLogViewHost(logViewHost);
            }

            if (ctx.containsKey("https_check")) {
                odps.getRestClient().setIgnoreCerts(!(Boolean) ctx.get("https_check"));
            }

        }

        if (context.containsKey("commandText")) {
            setCommandText((String) context.get("commandText"));
        }
    }

    /**
     * Parse old command line settings and set to new API accordingly.
     */
    private void parseOldCli(JobConf conf) {
        Properties prop = System.getProperties();

        //check for cost mode
        String cost = prop.getProperty(COST);
        if (cost != null && "true".equals(cost)) {
            defaultJob.setBoolean(ODPS_JOB_COST_ESTIMATE, true);
            setCostMode(true);
        }

        if (prop.getProperty(OLD_ENDPOINT_KEY) != null) {
            String endpoint = prop.getProperty(OLD_ENDPOINT_KEY);
            String project = prop.getProperty(OLD_PROJNAME_KEY);
            String accessId = prop.getProperty(OLD_ACCESSID_KEY);
            String accessKey = prop.getProperty(OLD_ACCESSKEY_KEY);
            String runmode = prop.getProperty(OLD_RUNMODE_KEY, "remote");

            if (runmode.equalsIgnoreCase("local")) {
                handleLocalMR(prop);
            } else {
                System.err.println("Running job in console.");
            }

            AccountProvider accountProvider = AccountProvider.ALIYUN;
            String apStr = prop.getProperty(OLD_ACCOUNT_PROVIDER_KEY);
            if (apStr != null) {
                apStr = apStr.trim().toUpperCase();
                try {
                    accountProvider = AccountProvider.valueOf(apStr);
                } catch (Exception exception) {
                    throw new RuntimeException("Unsupport account provider:" + apStr);
                }
            }

            Account account = null;
            switch (accountProvider) {
            case ALIYUN:
                account = new AliyunAccount(accessId, accessKey);
                break;

            default:
                throw new RuntimeException("unsupport account provider:" + accountProvider);
            }
            Odps odps = new Odps(account);
            odps.setDefaultProject(project);
            odps.setEndpoint(endpoint);
            setOdps(odps);
            setLocalRun(runmode.equalsIgnoreCase("local"));

            String resources = prop.getProperty(OLD_RESOURCE_KEY, "");
            String cpresources = prop.getProperty(OLD_CP_RESOURCE_KEY, "");
            if (!resources.isEmpty() && !cpresources.isEmpty()) {
                resources += ",";
            }
            resources += cpresources;
            conf.setResources(resources);
        }

        try {
            loadContextFile(prop);
        } catch (IOException e) {
            // Silently swallow it because adapter is not required to load context
            // file.
        }
    }

    /**
     * ??SessionState
     *
     * @return ?SessionState
     */
    public static SessionState get() {
        return tss.get();
    }

    /**
     * ?{@link com.aliyun.odps.Odps}
     *
     * @return {@link com.aliyun.odps.Odps}
     */
    public Odps getOdps() {
        return odps;
    }

    /**
     * {@link com.aliyun.odps.Odps}
     *
     * @param odps
     *     {@link com.aliyun.odps.Odps}
     */
    public void setOdps(Odps odps) {
        this.odps = odps;
        if (internalCli) {
            odps.getRestClient().setIgnoreCerts(true);
        }
    }

    /**
     * {@link com.aliyun.odps.mapred.conf.JobConf}
     * {@link com.aliyun.odps.mapred.conf.JobConf}?
     *
     * @return {@link com.aliyun.odps.mapred.conf.JobConf}
     */
    public JobConf getDefaultJob() {
        return defaultJob;
    }

    /**
     * local run?
     *
     * @return truelocal run?
     */
    public boolean isLocalRun() {
        return isLocalRun;
    }

    /**
     * local run?
     *
     * @param b
     *     true?local run
     */
    public void setLocalRun(boolean b) {
        isLocalRun = b;
    }

    /**
     * ??
     *
     * @param b
     *     true?
     */
    public void setCostMode(boolean b) {
        isCostMode = b;
    }

    /**
     * ??
     *
     * @return true?
     */
    public boolean isCostMode() {
        return isCostMode;
    }

    /**
     * {@link com.aliyun.odps.mapred.conf.JobConf}
     *
     * @param defaultJob
     */
    public void setDefaultJob(JobConf defaultJob) {
        this.defaultJob = defaultJob;
    }

    /**
     * Alias
     *
     * @param aliases
     */
    public void setAliases(Map<String, String> aliases) {
        this.aliases = aliases;
    }

    /**
     * Alias
     */
    public Map<String, String> getAliases() {
        return aliases;
    }

    private void handleLocalMR(Properties prop) {
        String tempDir = prop.getProperty(LOCAL_TEMP_DIR);
        if (tempDir != null && !tempDir.isEmpty()) {
            defaultJob.set(LOCAL_TEMP_DIR, tempDir);
        }

        String tempRetain = prop.getProperty(LOCAL_TEMP_RETAIN);
        if (tempRetain != null && tempRetain.trim().equalsIgnoreCase("false")) {
            defaultJob.setBoolean(LOCAL_TEMP_RETAIN, false);
        } else {
            defaultJob.setBoolean(LOCAL_TEMP_RETAIN, true);
        }

        String limitDownloadCount = prop.getProperty(LOCAL_RECORD_LIMIT);
        if (limitDownloadCount != null && !limitDownloadCount.isEmpty()) {
            try {
                int limit = Integer.parseInt(limitDownloadCount);
                if (limit > 0) {
                    defaultJob.setInt(LOCAL_RECORD_LIMIT, limit);
                }
            } catch (Exception e) {
            }
        }

        String downloadMode = prop.getProperty(LOCAL_DOWNLOAD_MODE, "AUTO");
        defaultJob.set(LOCAL_DOWNLOAD_MODE, downloadMode);

        String inputColumnSeperator = prop.getProperty(LOCAL_INPUT_COLUMN_SEPERATOR);
        if (inputColumnSeperator != null && !inputColumnSeperator.isEmpty()) {
            defaultJob.set(LOCAL_INPUT_COLUMN_SEPERATOR, inputColumnSeperator);
        }

        String outputColumnSeperator = prop.getProperty(LOCAL_OUTPUT_COLUMN_SEPERATOR);
        if (outputColumnSeperator != null && !outputColumnSeperator.isEmpty()) {
            defaultJob.set(LOCAL_OUTPUT_COLUMN_SEPERATOR, outputColumnSeperator);
        }

        boolean isSecurityEnabled = prop.getProperty(LOCAL_SECURITY_ENABLE, "false").equalsIgnoreCase("true");
        if (isSecurityEnabled) {
            boolean isJNIEnabled = prop.getProperty(LOCAL_SECURITY_JNI_ENABLE, "false").equalsIgnoreCase("true");
            String userDefinePolicy = prop.getProperty(LOCAL_USER_DEFINE_POLICY, "");
            defaultJob.setBoolean(LOCAL_SECURITY_ENABLE, isSecurityEnabled);
            defaultJob.setBoolean(LOCAL_SECURITY_JNI_ENABLE, isJNIEnabled);
            defaultJob.set(LOCAL_USER_DEFINE_POLICY, userDefinePolicy);
        }

    }

    public String getCommandText() {
        return commandText;
    }

    public void setCommandText(String commandText) {
        this.commandText = stripNonValidXMLCharacters(commandText);
    }

    /**
     * This method ensures that the output String has only valid XML unicode characters as specified
     * by the XML 1.0 standard. For reference, please see <a
     * href="http://www.w3.org/TR/2000/REC-xml-20001006#NT-Char">the standard</a>. This method will
     * return an empty String if the input is null or empty.
     * 
     * @param in The String whose non-valid characters we want to remove.
     * @return The in String, stripped of non-valid characters.
     */
    private String stripNonValidXMLCharacters(String in) {
        StringBuilder out = new StringBuilder();
        char current;
        if (in == null || ("".equals(in)))
            return "";
        for (int i = 0; i < in.length(); i++) {
            current = in.charAt(i);
            if ((current == 0x9) || (current == 0xA) || (current == 0xD)
                    || ((current >= 0x20) && (current <= 0xD7FF)) || ((current >= 0xE000) && (current <= 0xFFFD)))
                out.append(current);
        }
        return out.toString();
    }
}