Example usage for org.apache.hadoop.conf Configuration Configuration

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration Configuration.

Prototype

public Configuration() 

Source Link

Document

A new configuration.

Usage

From source file:azkaban.jobtype.connectors.teradata.HdfsToTeradataJobRunnerMain.java

License:Apache License

private HdfsToTeradataJobRunnerMain(Properties jobProps) throws FileNotFoundException, IOException {
    this(jobProps, new Whitelist(new Props(null, jobProps), FileSystem.get(new Configuration())),
            new Decryptions());
}

---

From source file:azkaban.jobtype.connectors.teradata.HdfsToTeradataJobRunnerMain.java

License:Apache License

@VisibleForTesting
HdfsToTeradataJobRunnerMain(Properties jobProps, Whitelist whitelist, Decryptions decryptions)
        throws FileNotFoundException, IOException {
    _logger = JobUtils.initJobLogger();/*from w  w w  .  jav  a2s  .  c o  m*/
    _jobProps = jobProps;

    Props props = new Props(null, _jobProps);

    HadoopConfigurationInjector.injectResources(props);
    Configuration conf = new Configuration();
    UserGroupInformation.setConfiguration(conf);

    if (props.containsKey(Whitelist.WHITE_LIST_FILE_PATH_KEY)) {
        whitelist.validateWhitelisted(props);
    }

    String encryptedCredential = _jobProps.getProperty(TdchConstants.TD_ENCRYPTED_CREDENTIAL_KEY);
    String cryptoKeyPath = _jobProps.getProperty(TdchConstants.TD_CRYPTO_KEY_PATH_KEY);
    String password = null;

    if (encryptedCredential != null && cryptoKeyPath != null) {
        password = decryptions.decrypt(encryptedCredential, cryptoKeyPath, FileSystem.get(new Configuration()));
    }

    _params = TdchParameters.builder().mrParams(_jobProps.getProperty(TdchConstants.HADOOP_CONFIG_KEY))
            .libJars(props.getString(TdchConstants.LIB_JARS_KEY))
            .tdJdbcClassName(TdchConstants.TERADATA_JDBCDRIVER_CLASSNAME)
            .teradataHostname(props.getString(TdchConstants.TD_HOSTNAME_KEY))
            .fileFormat(_jobProps.getProperty(TdchConstants.HDFS_FILE_FORMAT_KEY))
            .fieldSeparator(_jobProps.getProperty(TdchConstants.HDFS_FIELD_SEPARATOR_KEY))
            .jobType(TdchConstants.TDCH_JOB_TYPE).userName(props.getString(TdchConstants.TD_USERID_KEY))
            .credentialName(_jobProps.getProperty(TdchConstants.TD_CREDENTIAL_NAME_KEY)).password(password)
            .avroSchemaPath(_jobProps.getProperty(TdchConstants.AVRO_SCHEMA_PATH_KEY))
            .avroSchemaInline(_jobProps.getProperty(TdchConstants.AVRO_SCHEMA_INLINE_KEY))
            .sourceHdfsPath(props.getString(TdchConstants.SOURCE_HDFS_PATH_KEY))
            .targetTdTableName(props.getString(TdchConstants.TARGET_TD_TABLE_NAME_KEY))
            .errorTdDatabase(_jobProps.getProperty(TdchConstants.ERROR_DB_KEY))
            .errorTdTableName(_jobProps.getProperty(TdchConstants.ERROR_TABLE_KEY))
            .tdInsertMethod(_jobProps.getProperty(TdchConstants.TD_INSERT_METHOD_KEY))
            .numMapper(TdchConstants.DEFAULT_NO_MAPPERS)
            .otherProperties(_jobProps.getProperty(TdchConstants.TD_OTHER_PROPERTIES_HOCON_KEY)).build();
}

---

From source file:azkaban.jobtype.connectors.teradata.TeradataToHdfsJobRunnerMain.java

License:Apache License

public TeradataToHdfsJobRunnerMain() throws FileNotFoundException, IOException {
    _logger = JobUtils.initJobLogger();//from   w w  w. j  a v a2 s. co m
    _jobProps = HadoopSecureWrapperUtils.loadAzkabanProps();

    Props props = new Props(null, _jobProps);
    HadoopConfigurationInjector.injectResources(props);
    Configuration conf = new Configuration();
    UserGroupInformation.setConfiguration(conf);

    if (props.containsKey(Whitelist.WHITE_LIST_FILE_PATH_KEY)) {
        new Whitelist(props, FileSystem.get(conf)).validateWhitelisted(props);
    }
    String encryptedCredential = _jobProps.getProperty(TdchConstants.TD_ENCRYPTED_CREDENTIAL_KEY);
    String cryptoKeyPath = _jobProps.getProperty(TdchConstants.TD_CRYPTO_KEY_PATH_KEY);
    String password = null;
    if (encryptedCredential != null && cryptoKeyPath != null) {
        password = new Decryptions().decrypt(encryptedCredential, cryptoKeyPath,
                FileSystem.get(new Configuration()));
    }

    _params = TdchParameters.builder().mrParams(_jobProps.getProperty(TdchConstants.HADOOP_CONFIG_KEY))
            .libJars(props.getString(TdchConstants.LIB_JARS_KEY))
            .tdJdbcClassName(TdchConstants.TERADATA_JDBCDRIVER_CLASSNAME)
            .teradataHostname(props.getString(TdchConstants.TD_HOSTNAME_KEY))
            .fileFormat(_jobProps.getProperty(TdchConstants.HDFS_FILE_FORMAT_KEY))
            .fieldSeparator(_jobProps.getProperty(TdchConstants.HDFS_FIELD_SEPARATOR_KEY))
            .jobType(TdchConstants.TDCH_JOB_TYPE).userName(props.getString(TdchConstants.TD_USERID_KEY))
            .credentialName(_jobProps.getProperty(TdchConstants.TD_CREDENTIAL_NAME_KEY)).password(password)
            .avroSchemaPath(_jobProps.getProperty(TdchConstants.AVRO_SCHEMA_PATH_KEY))
            .avroSchemaInline(_jobProps.getProperty(TdchConstants.AVRO_SCHEMA_INLINE_KEY))
            .sourceTdTableName(_jobProps.getProperty(TdchConstants.SOURCE_TD_TABLE_NAME_KEY))
            .sourceQuery(_jobProps.getProperty(TdchConstants.SOURCE_TD_QUERY_NAME_KEY))
            .targetHdfsPath(props.getString(TdchConstants.TARGET_HDFS_PATH_KEY))
            .tdRetrieveMethod(_jobProps.getProperty(TdchConstants.TD_RETRIEVE_METHOD_KEY))
            .numMapper(TdchConstants.DEFAULT_NO_MAPPERS).build();
}

---

From source file:azkaban.jobtype.connectors.TeradataToHdfsJobRunnerMain.java

License:Apache License

public TeradataToHdfsJobRunnerMain() throws FileNotFoundException, IOException {
    _logger = JobUtils.initJobLogger();//  w  ww  .j  a v a2  s .  c  om
    _jobProps = HadoopSecureWrapperUtils.loadAzkabanProps();

    Props props = new Props(null, _jobProps);
    HadoopConfigurationInjector.injectResources(props);
    UserGroupInformation.setConfiguration(new Configuration());

    _params = TdchParameters.builder().mrParams(_jobProps.getProperty(TdchConstants.HADOOP_CONFIG_KEY))
            .libJars(props.getString(TdchConstants.LIB_JARS_KEY))
            .tdJdbcClassName(TdchConstants.TERADATA_JDBCDRIVER_CLASSNAME)
            .teradataHostname(props.getString(TdchConstants.TD_HOSTNAME_KEY))
            .fileFormat(_jobProps.getProperty(TdchConstants.HDFS_FILE_FORMAT_KEY))
            .fieldSeparator(_jobProps.getProperty(TdchConstants.HDFS_FIELD_SEPARATOR_KEY))
            .jobType(TdchConstants.TDCH_JOB_TYPE).userName(props.getString(TdchConstants.TD_USERID_KEY))
            .credentialName(String.format(TdchConstants.TD_WALLET_FORMAT,
                    props.getString(TdchConstants.TD_CREDENTIAL_NAME)))
            .avroSchemaPath(_jobProps.getProperty(TdchConstants.AVRO_SCHEMA_PATH_KEY))
            .avroSchemaInline(_jobProps.getProperty(TdchConstants.AVRO_SCHEMA_INLINE_KEY))
            .sourceTdTableName(_jobProps.getProperty(TdchConstants.SOURCE_TD_TABLE_NAME_KEY))
            .sourceQuery(_jobProps.getProperty(TdchConstants.SOURCE_TD_QUERY_NAME_KEY))
            .targetHdfsPath(props.getString(TdchConstants.TARGET_HDFS_PATH_KEY))
            .tdRetrieveMethod(_jobProps.getProperty(TdchConstants.TD_RETRIEVE_METHOD_KEY))
            .numMapper(DEFAULT_NO_MAPPERS).build();
}

---

From source file:azkaban.jobtype.HadoopJavaJobRunnerMain.java

License:Apache License

public HadoopJavaJobRunnerMain() throws Exception {
    Runtime.getRuntime().addShutdownHook(new Thread() {
        @Override/* w  w w . jav  a 2  s.c o  m*/
        public void run() {
            cancelJob();
        }
    });

    try {
        _jobName = System.getenv(ProcessJob.JOB_NAME_ENV);
        String propsFile = System.getenv(ProcessJob.JOB_PROP_ENV);

        _logger = Logger.getRootLogger();
        _logger.removeAllAppenders();
        ConsoleAppender appender = new ConsoleAppender(DEFAULT_LAYOUT);
        appender.activateOptions();
        _logger.addAppender(appender);
        _logger.setLevel(Level.INFO); //Explicitly setting level to INFO

        Properties props = new Properties();
        props.load(new BufferedReader(new FileReader(propsFile)));

        HadoopConfigurationInjector.injectResources(new Props(null, props));

        final Configuration conf = new Configuration();

        UserGroupInformation.setConfiguration(conf);
        securityEnabled = UserGroupInformation.isSecurityEnabled();

        _logger.info("Running job " + _jobName);
        String className = props.getProperty(JOB_CLASS);
        if (className == null) {
            throw new Exception("Class name is not set.");
        }
        _logger.info("Class name " + className);

        UserGroupInformation loginUser = null;
        UserGroupInformation proxyUser = null;

        if (shouldProxy(props)) {
            String userToProxy = props.getProperty("user.to.proxy");
            if (securityEnabled) {
                String filelocation = System.getenv(HADOOP_TOKEN_FILE_LOCATION);
                _logger.info("Found token file " + filelocation);
                _logger.info("Security enabled is " + UserGroupInformation.isSecurityEnabled());

                _logger.info("Setting mapreduce.job.credentials.binary to " + filelocation);
                System.setProperty("mapreduce.job.credentials.binary", filelocation);

                _logger.info("Proxying enabled.");

                loginUser = UserGroupInformation.getLoginUser();

                _logger.info("Current logged in user is " + loginUser.getUserName());

                proxyUser = UserGroupInformation.createProxyUser(userToProxy, loginUser);
                for (Token<?> token : loginUser.getTokens()) {
                    proxyUser.addToken(token);
                }
            } else {
                proxyUser = UserGroupInformation.createRemoteUser(userToProxy);
            }
            _logger.info("Proxied as user " + userToProxy);
        }

        // Create the object using proxy
        if (shouldProxy(props)) {
            _javaObject = getObjectAsProxyUser(props, _logger, _jobName, className, proxyUser);
        } else {
            _javaObject = getObject(_jobName, className, props, _logger);
        }

        if (_javaObject == null) {
            _logger.info("Could not create java object to run job: " + className);
            throw new Exception("Could not create running object");
        }
        _logger.info("Got object " + _javaObject.toString());

        _cancelMethod = props.getProperty(CANCEL_METHOD_PARAM, DEFAULT_CANCEL_METHOD);

        final String runMethod = props.getProperty(RUN_METHOD_PARAM, DEFAULT_RUN_METHOD);
        _logger.info("Invoking method " + runMethod);

        if (shouldProxy(props)) {
            _logger.info("Proxying enabled.");
            runMethodAsUser(props, _javaObject, runMethod, proxyUser);
        } else {
            _logger.info("Proxy check failed, not proxying run.");
            runMethod(_javaObject, runMethod);
        }

        _isFinished = true;

        // Get the generated properties and store them to disk, to be read
        // by ProcessJob.
        try {
            final Method generatedPropertiesMethod = _javaObject.getClass()
                    .getMethod(GET_GENERATED_PROPERTIES_METHOD, new Class<?>[] {});
            Object outputGendProps = generatedPropertiesMethod.invoke(_javaObject, new Object[] {});

            if (outputGendProps != null) {
                final Method toPropertiesMethod = outputGendProps.getClass().getMethod("toProperties",
                        new Class<?>[] {});
                Properties properties = (Properties) toPropertiesMethod.invoke(outputGendProps,
                        new Object[] {});

                Props outputProps = new Props(null, properties);
                outputGeneratedProperties(outputProps);
            } else {
                _logger.info(GET_GENERATED_PROPERTIES_METHOD
                        + " method returned null.  No properties to pass along");
            }
        } catch (NoSuchMethodException e) {
            _logger.info(String.format(
                    "Apparently there isn't a method[%s] on object[%s], using " + "empty Props object instead.",
                    GET_GENERATED_PROPERTIES_METHOD, _javaObject));
            outputGeneratedProperties(new Props());
        }
    } catch (Exception e) {
        _isFinished = true;
        throw e;
    }
}

---

From source file:azkaban.jobtype.HadoopJavaJobRunnerMain.java

License:Apache License

private void runMethodAsUser(Properties props, final Object obj, final String runMethod,
        final UserGroupInformation ugi) throws IOException, InterruptedException {
    ugi.doAs(new PrivilegedExceptionAction<Void>() {
        @Override/*from   w w w . ja v  a  2s . c  o m*/
        public Void run() throws Exception {

            Configuration conf = new Configuration();
            if (System.getenv(HADOOP_TOKEN_FILE_LOCATION) != null) {
                conf.set(MAPREDUCE_JOB_CREDENTIALS_BINARY, System.getenv(HADOOP_TOKEN_FILE_LOCATION));
            }

            runMethod(obj, runMethod);
            return null;
        }
    });
}

---

From source file:azkaban.jobtype.HadoopSecureSparkWrapper.java

License:Apache License

/**
 * Entry point: a Java wrapper to the spark-submit command
 * //from w  w  w  .j a  va  2s. c  o m
 * @param args
 * @throws Exception
 */
public static void main(final String[] args) throws Exception {

    Properties jobProps = HadoopSecureWrapperUtils.loadAzkabanProps();
    HadoopConfigurationInjector.injectResources(new Props(null, jobProps));

    if (HadoopSecureWrapperUtils.shouldProxy(jobProps)) {
        String tokenFile = System.getenv(HADOOP_TOKEN_FILE_LOCATION);
        UserGroupInformation proxyUser = HadoopSecureWrapperUtils.setupProxyUser(jobProps, tokenFile, logger);
        proxyUser.doAs(new PrivilegedExceptionAction<Void>() {
            @Override
            public Void run() throws Exception {
                runSpark(args, new Configuration());
                return null;
            }
        });
    } else {
        runSpark(args, new Configuration());
    }
}

---

From source file:azkaban.jobtype.HadoopSecureWrapperUtils.java

License:Apache License

/**
 * Sets up the UserGroupInformation proxyUser object so that calling code can do doAs returns null
 * if the jobProps does not call for a proxyUser
 * /*from   w  ww  . j  a  va  2s  . co m*/
 * @param jobPropsIn
 * @param tokenFile
 *          pass tokenFile if known. Pass null if the tokenFile is in the environmental variable
 *          already.
 * @param log
 * @return returns null if no need to run as proxyUser, otherwise returns valid proxyUser that can
 *         doAs
 */
public static UserGroupInformation setupProxyUser(Properties jobProps, String tokenFile, Logger log) {
    UserGroupInformation proxyUser = null;

    if (!HadoopSecureWrapperUtils.shouldProxy(jobProps)) {
        log.info("submitting job as original submitter, not proxying");
        return proxyUser;
    }

    // set up hadoop related configurations
    final Configuration conf = new Configuration();
    UserGroupInformation.setConfiguration(conf);
    boolean securityEnabled = UserGroupInformation.isSecurityEnabled();

    // setting up proxy user if required
    try {
        String userToProxy = null;
        userToProxy = jobProps.getProperty(HadoopSecurityManager.USER_TO_PROXY);
        if (securityEnabled) {
            proxyUser = HadoopSecureWrapperUtils.createSecurityEnabledProxyUser(userToProxy, tokenFile, log);
            log.info("security enabled, proxying as user " + userToProxy);
        } else {
            proxyUser = UserGroupInformation.createRemoteUser(userToProxy);
            log.info("security not enabled, proxying as user " + userToProxy);
        }
    } catch (IOException e) {
        log.error("HadoopSecureWrapperUtils.setupProxyUser threw an IOException", e);
    }

    return proxyUser;
}

---

From source file:azkaban.jobtype.hiveutils.azkaban.hive.actions.DropAllPartitionsAddLatest.java

License:Apache License

@Override
public void execute() throws HiveViaAzkabanException {
    ArrayList<HQL> hql = new ArrayList<HQL>();
    hql.add(new UseDatabaseHQL(database));

    Configuration conf = new Configuration();
    try {//from www. ja  va2 s  . c  om
        FileSystem fs = FileSystem.get(conf);

        for (String table : tables) {
            LOG.info("Determining HQL commands for table " + table);
            hql.addAll(addAndDrop(fs, tableLocations, table));
        }
        fs.close();
    } catch (IOException e) {
        throw new HiveViaAzkabanException("Exception fetching the directories/partitions from HDFS", e);
    }

    StringBuffer query = new StringBuffer();
    for (HQL q : hql) {
        query.append(q.toHQL()).append("\n");
    }

    System.out.println("Query to execute:\n" + query.toString());
    try {
        hqe.executeQuery(query.toString());
    } catch (HiveQueryExecutionException e) {
        throw new HiveViaAzkabanException("Problem executing query [" + query.toString() + "] on Hive", e);
    }

}

---

From source file:azkaban.jobtype.hiveutils.azkaban.hive.actions.UpdateTableLocationToLatest.java

License:Apache License

@Override
public void execute() throws HiveViaAzkabanException {
    ArrayList<HQL> hql = new ArrayList<HQL>();
    hql.add(new UseDatabaseHQL(database));

    Configuration conf = new Configuration();
    try {//from   w  w w  .ja  va2  s  .com
        FileSystem fs = FileSystem.get(conf);

        for (int i = 0; i < tables.length; i++) {
            LOG.info("Determining HQL commands for table " + tables[i]);
            hql.add(latestURI(fs, tablesLocations[i], tables[i]));
        }
        fs.close();
    } catch (IOException e) {
        throw new HiveViaAzkabanException("Exception fetching the directories from HDFS", e);
    }
    StringBuffer query = new StringBuffer();
    for (HQL q : hql) {
        query.append(q.toHQL()).append("\n");
    }

    System.out.println("Query to execute:\n" + query.toString());
    try {
        hqe.executeQuery(query.toString());
    } catch (HiveQueryExecutionException e) {
        throw new HiveViaAzkabanException("Problem executing query [" + query.toString() + "] on Hive", e);
    }

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »