org.chombo.util.ConfigurationLoader.java Source code

Java tutorial

Introduction

Here is the source code for org.chombo.util.ConfigurationLoader.java

Source

/*
 * chombo: Hadoop Map Reduce utility
 * Author: Pranab Ghosh
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License. You may
 * obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0 
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

package org.chombo.util;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import com.amazonaws.auth.PropertiesCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3Object;

/**
 * Sets configuration. Global config params are always loaded. Specific config group 
 * params as identified by a key prefix are also loaded. A version can also be specified
 * for the config group being loaded. A group will correspond to configuration parameters
 * for a map reduce job
 * 
 * @author pranab
 *
 */
public class ConfigurationLoader {
    private Configuration conf;
    private String project;
    private List<String> groups = new ArrayList<String>();
    private int version;

    private final String CONF_FILE_PROP_NAME = "conf.path";
    private final String CONF_GROUP = "conf.group";

    private final String FS_DEF_CONFIG_DIR = "/var/mawazo/";
    private final String HDFS_DEF_CONFIG_DIR = "/var/mawazo/";
    private final String HDFS_PREFIX = "hdfs:";
    private final int HDFS_PREFIX_LEN = 5;
    private final String S3_PREFIX = "s3n:";
    private final String PROP_FILE_EXT = ".properties";
    private Pattern s3pattern = Pattern.compile("s3n:/+([^/]+)/+(.*)");
    private final String GLOBAL_KEY_PREFIX = "global";
    private int globalKeyPrefixLen = GLOBAL_KEY_PREFIX.length();

    /**
     * @param conf
     * @param project
     * @param keyPrefix
     */
    public ConfigurationLoader(Configuration conf, String project) {
        super();
        this.conf = conf;
        this.project = project;

        String confGroup = conf.get(CONF_GROUP);
        if (null == confGroup) {
            throw new IllegalStateException("configuration group not provided");
        }

        if (confGroup.contains(Utility.configDelim)) {
            //multiple groups and/or higher version
            String[] items = confGroup.split(Utility.configDelim);

            //group names and end with version
            for (int i = 0; i < items.length - 1; ++i) {
                groups.add(items[i]);
            }
            version = Integer.parseInt(items[items.length - 1]);
        } else {
            //only one group with default version of 0
            groups.add(confGroup);
        }
    }

    /**
      * @param conf
      * @param project
      * @throws Exception
      */
    public void set() throws Exception {
        boolean found = false;
        String confFilePath = conf.get(CONF_FILE_PROP_NAME);

        //user provided config file path
        if (null != confFilePath) {
            if (confFilePath.startsWith(S3_PREFIX)) {
                loadConfigS3(confFilePath);
                System.out.println("config found in user specified Amazon S3 file");
            } else if (confFilePath.startsWith(HDFS_PREFIX)) {
                loadConfigHdfs(confFilePath.substring(HDFS_PREFIX_LEN));
                System.out.println("config found in user specified HDFS file");
            } else {
                loadConfig(confFilePath, false);
                System.out.println("config found in user specified FS  file");
            }
        } else {
            //default file system path
            confFilePath = FS_DEF_CONFIG_DIR + project + PROP_FILE_EXT;
            found = loadConfig(confFilePath, true);

            //default HDFS path
            if (!found) {
                confFilePath = HDFS_DEF_CONFIG_DIR + project + PROP_FILE_EXT;
                loadConfigHdfs(confFilePath);
                System.out.println("config found in default HDFS location");
            } else {
                System.out.println("config found in default FS location");
            }
        }
    }

    /**
     * @param conf
     * @param confFilePath
     * @param handleErr
     * @return
     * @throws IOException
     */
    private boolean loadConfig(String confFilePath, boolean handleErr) throws IOException {
        boolean found = false;

        try {
            FileInputStream fis = new FileInputStream(confFilePath);
            List<String> lines = getConfigLines(fis);
            setFilteredConfiguration(lines);
            found = true;
        } catch (FileNotFoundException ex) {
            if (!handleErr) {
                throw ex;
            }
        }
        return found;
    }

    /**
     * @param conf
     * @param confFilePath
     * @return
     * @throws IOException
     */
    private boolean loadConfigHdfs(String confFilePath) throws IOException {
        boolean found = false;

        FileSystem dfs = FileSystem.get(conf);
        Path src = new Path(confFilePath);
        FSDataInputStream fis = dfs.open(src);
        List<String> lines = getConfigLines(fis);
        setFilteredConfiguration(lines);
        found = true;
        return found;
    }

    /**
     * @param confFilePath
     * @return
     * @throws IOException
     */
    private boolean loadConfigS3(String confFilePath) throws IOException {
        Matcher matcher = s3pattern.matcher(confFilePath);
        matcher.matches();
        String bucket = matcher.group(1);
        String key = matcher.group(2);
        AmazonS3 s3 = new AmazonS3Client(
                new PropertiesCredentials(Utility.class.getResourceAsStream("AwsCredentials.properties")));

        S3Object object = s3.getObject(new GetObjectRequest(bucket, key));
        InputStream fis = object.getObjectContent();
        List<String> lines = getConfigLines(fis);
        setFilteredConfiguration(lines);
        return true;
    }

    /**
     * @param lines
     */
    private void setFilteredConfiguration(List<String> lines) {
        Map<String, Pair<String, Integer>> multiVersionConfig = new HashMap<String, Pair<String, Integer>>();

        System.out.println("configuration settings:");
        for (String line : lines) {
            if (!line.isEmpty() && !line.startsWith("#")) {
                String[] items = line.split("=");
                if (items.length != 2) {
                    throw new IllegalArgumentException("invalid cofig file format");
                }
                items[0] = items[0].trim();
                items[1] = items[1].trim();

                String valSt = items[1];
                if (items[0].startsWith(GLOBAL_KEY_PREFIX)) {
                    //global config
                    String keySt = items[0].substring(globalKeyPrefixLen + 1);
                    conf.set(keySt, valSt);
                    System.out.println(keySt + "\\t" + valSt);
                } else if (shouldInclude(items[0])) {
                    //specific config group
                    String keySt = items[0];
                    Pair<String, Integer> versionedVal = multiVersionConfig.get(keySt);
                    if (null == versionedVal) {
                        versionedVal = new Pair<String, Integer>(valSt, -1);
                    }

                    //keep replacing until we have reached the right version
                    if (versionedVal.getRight() < version) {
                        multiVersionConfig.put(keySt,
                                new Pair<String, Integer>(valSt, versionedVal.getRight() + 1));
                    }
                }
            }
        }

        //nothing found
        if (multiVersionConfig.isEmpty()) {
            throw new IllegalStateException("no configuration parameter found for groups " + groups);
        }

        //set config with right version
        for (String key : multiVersionConfig.keySet()) {
            String val = multiVersionConfig.get(key).getLeft();
            conf.set(key, val);
            System.out.println(key + "\\t" + val);
        }
    }

    /**
     * @param key
     * @return
     */
    private boolean shouldInclude(String key) {
        boolean include = false;
        for (String group : groups) {
            if (key.startsWith(group)) {
                include = true;
                break;
            }
        }

        return include;
    }

    /**
     * @param fs
     * @return
     * @throws IOException
     */
    private List<String> getConfigLines(InputStream inStr) throws IOException {
        BufferedReader reader = new BufferedReader(new InputStreamReader(inStr));
        String line = null;
        List<String> lines = new ArrayList<String>();
        while ((line = reader.readLine()) != null) {
            lines.add(line);
        }

        return lines;
    }

}