Example usage for org.apache.hadoop.conf Configuration getStrings

List of usage examples for org.apache.hadoop.conf Configuration getStrings

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. getStrings

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getStrings.

Prototype

public String[] getStrings(String name)

Source Link

Document

Get the comma delimited values of the name property as an array of Strings.

Usage

From source file:org.apache.nutchbase.parse.html.DOMContentUtils.java

License:Apache License

public void setConf(Configuration conf) {
    // forceTags is used to override configurable tag ignoring, later on
    Collection<String> forceTags = new ArrayList<String>(1);

    this.conf = conf;
    linkParams.clear();/*from w w w.j ava 2s  . c  o m*/
    linkParams.put("a", new LinkParams("a", "href", 1));
    linkParams.put("area", new LinkParams("area", "href", 0));
    if (conf.getBoolean("parser.html.form.use_action", true)) {
        linkParams.put("form", new LinkParams("form", "action", 1));
        if (conf.get("parser.html.form.use_action") != null)
            forceTags.add("form");
    }
    linkParams.put("frame", new LinkParams("frame", "src", 0));
    linkParams.put("iframe", new LinkParams("iframe", "src", 0));
    linkParams.put("script", new LinkParams("script", "src", 0));
    linkParams.put("link", new LinkParams("link", "href", 0));
    linkParams.put("img", new LinkParams("img", "src", 0));

    // remove unwanted link tags from the linkParams map
    String[] ignoreTags = conf.getStrings("parser.html.outlinks.ignore_tags");
    for (int i = 0; ignoreTags != null && i < ignoreTags.length; i++) {
        if (!forceTags.contains(ignoreTags[i]))
            linkParams.remove(ignoreTags[i]);
    }
    this.ignoreNoFollow = conf.getBoolean("parser.html.outlinks.ignore_nofollow", false);
}

From source file:org.apache.oozie.action.hadoop.HDFSCredentials.java

License:Apache License

/**
 * Add an HDFS_DELEGATION_TOKEN to the {@link Credentials} provided.
 * This is also important to ensure that log aggregation works correctly from the NM
 *
 * @param credentials the credentials object which is updated
 * @param config launcher AM configuration
 * @param props properties for getting credential token or certificate
 * @param context workflow context//from   ww w .j a v  a  2 s. com
 * @throws Exception thrown if failed
 */
@Override
public void updateCredentials(Credentials credentials, Configuration config, CredentialsProperties props,
        ActionExecutor.Context context) throws Exception {
    final String jobNameNodes[] = config.getStrings(MRJobConfig.JOB_NAMENODES);
    if (jobNameNodes != null) {
        final Path[] paths = new Path[jobNameNodes.length];
        for (int i = 0; i != jobNameNodes.length; ++i) {
            paths[i] = new Path(jobNameNodes[i]);
        }

        final UserGroupInformation ugi = Services.get().get(UserGroupInformationService.class)
                .getProxyUser(context.getWorkflow().getUser());
        obtainTokensForNamenodes(credentials, config, ugi, paths);
    } else {
        obtainTokenForAppFileSystemNameNode(credentials, config, context);
    }

}

From source file:org.apache.oozie.action.hadoop.JavaActionExecutor.java

License:Apache License

protected void addActionLibs(Path appPath, Configuration conf) throws ActionExecutorException {
    String[] actionLibsStrArr = conf.getStrings("oozie.launcher.oozie.libpath");
    if (actionLibsStrArr != null) {
        try {//from w  ww.  j a v  a2  s  . c  om
            for (String actionLibsStr : actionLibsStrArr) {
                actionLibsStr = actionLibsStr.trim();
                if (actionLibsStr.length() > 0) {
                    Path actionLibsPath = new Path(actionLibsStr);
                    String user = conf.get("user.name");
                    FileSystem fs = Services.get().get(HadoopAccessorService.class).createFileSystem(user,
                            appPath.toUri(), conf);
                    if (fs.exists(actionLibsPath)) {
                        FileStatus[] files = fs.listStatus(actionLibsPath);
                        for (FileStatus file : files) {
                            addToCache(conf, appPath, file.getPath().toUri().getPath(), false);
                        }
                    }
                }
            }
        } catch (HadoopAccessorException ex) {
            throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED,
                    ex.getErrorCode().toString(), ex.getMessage());
        } catch (IOException ex) {
            throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED,
                    "It should never happen", ex.getMessage());
        }
    }
}

From source file:org.apache.oozie.action.hadoop.JavaActionExecutor.java

License:Apache License

@SuppressWarnings("unchecked")
public void setLibFilesArchives(Context context, Element actionXml, Path appPath, Configuration conf)
        throws ActionExecutorException {
    Configuration proto = context.getProtoActionConf();

    // Workflow lib/
    String[] paths = proto.getStrings(WorkflowAppService.APP_LIB_PATH_LIST);
    if (paths != null) {
        for (String path : paths) {
            addToCache(conf, appPath, path, false);
        }// w  w w .j  a v a2 s.  com
    }

    // Action libs
    addActionLibs(appPath, conf);

    // files and archives defined in the action
    for (Element eProp : (List<Element>) actionXml.getChildren()) {
        if (eProp.getName().equals("file")) {
            String[] filePaths = eProp.getTextTrim().split(",");
            for (String path : filePaths) {
                addToCache(conf, appPath, path.trim(), false);
            }
        } else if (eProp.getName().equals("archive")) {
            String[] archivePaths = eProp.getTextTrim().split(",");
            for (String path : archivePaths) {
                addToCache(conf, appPath, path.trim(), true);
            }
        }
    }

    addAllShareLibs(appPath, conf, context, actionXml);
}

From source file:org.apache.oozie.action.hadoop.JavaActionExecutor.java

License:Apache License

/**
 * Return the sharelib names for the action.
 * <p>/*  w  w  w .j  ava  2  s .  c  o m*/
 * If <code>NULL</code> or empty, it means that the action does not use the action
 * sharelib.
 * <p>
 * If a non-empty string, i.e. <code>foo</code>, it means the action uses the
 * action sharelib sub-directory <code>foo</code> and all JARs in the sharelib
 * <code>foo</code> directory will be in the action classpath. Multiple sharelib
 * sub-directories can be specified as a comma separated list.
 * <p>
 * The resolution is done using the following precedence order:
 * <ul>
 *     <li><b>action.sharelib.for.#ACTIONTYPE#</b> in the action configuration</li>
 *     <li><b>action.sharelib.for.#ACTIONTYPE#</b> in the job configuration</li>
 *     <li><b>action.sharelib.for.#ACTIONTYPE#</b> in the oozie configuration</li>
 *     <li>Action Executor <code>getDefaultShareLibName()</code> method</li>
 * </ul>
 *
 *
 * @param context executor context.
 * @param actionXml
 * @param conf action configuration.
 * @return the action sharelib names.
 */
protected String[] getShareLibNames(Context context, Element actionXml, Configuration conf) {
    String[] names = conf.getStrings(ACTION_SHARELIB_FOR + getType());
    if (names == null || names.length == 0) {
        try {
            XConfiguration jobConf = new XConfiguration(new StringReader(context.getWorkflow().getConf()));
            names = jobConf.getStrings(ACTION_SHARELIB_FOR + getType());
            if (names == null || names.length == 0) {
                names = Services.get().getConf().getStrings(ACTION_SHARELIB_FOR + getType());
                if (names == null || names.length == 0) {
                    String name = getDefaultShareLibName(actionXml);
                    if (name != null) {
                        names = new String[] { name };
                    }
                }
            }
        } catch (IOException ex) {
            throw new RuntimeException("It cannot happen, " + ex.toString(), ex);
        }
    }
    return names;
}

From source file:org.apache.oozie.action.hadoop.SharelibResolver.java

License:Apache License

/**
 * Return the sharelib names for the action based on the given configurations.
 * <p>/*  www  .j  a  va  2  s .  co  m*/
 * If <code>NULL</code> or empty, it means that the action does not use the action
 * sharelib.
 * <p>
 * If a non-empty string, i.e. <code>foo</code>, it means the action uses the
 * action sharelib sub-directory <code>foo</code> and all JARs in the sharelib
 * <code>foo</code> directory will be in the action classpath. Multiple sharelib
 * sub-directories can be specified as a comma separated list.
 * <p>
 * The resolution is done using the following precedence order:
 * <ul>
 *     <li><b><sharelib></b> tag in the action's launcher configuration</li>
 *     <li><b>oozie.action.sharelib.for.#ACTIONTYPE#</b> in the action configuration</li>
 *     <li><b><sharelib></b> tag in the workflow's launcher configuration</li>
 *     <li><b>oozie.action.sharelib.for.#ACTIONTYPE#</b> in the workflow configuration</li>
 *     <li><b>oozie.action.sharelib.for.#ACTIONTYPE#</b> in the oozie configuration</li>
 *     <li>Action Executor <code>getDefaultShareLibName()</code> method</li>
 * </ul>
 *
 * @param sharelibPropertyName the property for the current sharelib. E.g. oozie.action.sharelib.for.java
 * @param actionConf the configuration of the current action
 * @param workflowConf the global configuration for the workflow
 * @param oozieServerConfiguration the Oozie server's configuration
 * @param defaultValue the default value to use if there is no sharelib definition in the configs
 */
SharelibResolver(final String sharelibPropertyName, final Configuration actionConf,
        final XConfiguration workflowConf, final Configuration oozieServerConfiguration,
        final String defaultValue) {
    if (defaultValue == null) {
        this.defaultValue = new String[0];
    } else {
        this.defaultValue = new String[] { defaultValue };
    }
    configProviders = Arrays.asList(() -> actionConf.getStrings(LauncherAM.OOZIE_LAUNCHER_SHARELIB_PROPERTY),
            () -> actionConf.getStrings(sharelibPropertyName),
            () -> workflowConf.getStrings(LauncherAM.OOZIE_LAUNCHER_SHARELIB_PROPERTY),
            () -> workflowConf.getStrings(sharelibPropertyName),
            () -> oozieServerConfiguration.getStrings(sharelibPropertyName));
}

From source file:org.apache.oozie.compression.CodecFactory.java

License:Apache License

/**
 * Initialize the codec factory to maintain list of codecs
 * @param conf/*from  w  w w. j  a  va2  s.co m*/
 * @throws Exception
 */
public static void initialize(Configuration conf) throws Exception {
    String outputCompressionStr = conf.get(COMPRESSION_OUTPUT_CODEC);
    if (outputCompressionStr == null || outputCompressionStr.trim().equalsIgnoreCase("NONE")
            || outputCompressionStr.trim().equalsIgnoreCase("")) {
        isEnabled = false;
    } else {
        outputCompressionStr = outputCompressionStr.trim();
        isEnabled = true;
    }
    String[] outputCompressionCodecs = conf.getStrings(COMPRESSION_CODECS);
    for (String comp : outputCompressionCodecs) {
        parseCompressionConfig(comp);
    }
    if (isEnabled) {
        if (REGISTERED.get(GzipCompressionCodec.CODEC_NAME) == null) {
            REGISTERED.put(GzipCompressionCodec.CODEC_NAME, new GzipCompressionCodec());
        }
        outputCompressionCodec = REGISTERED.get(outputCompressionStr);
        if (outputCompressionCodec == null) {
            throw new RuntimeException("No codec class found for codec " + outputCompressionStr);
        }
    }
    LOG.info("Using " + outputCompressionStr + " as output compression codec");

    // Initialize header bytes
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream daos = new DataOutputStream(baos);
    // magic data
    daos.write(COMPRESSION_MAGIC_DATA.getBytes(UTF_8_ENCODING));
    // version
    daos.writeInt(1);
    // no of key value pairs
    daos.writeInt(1);
    daos.writeUTF(COMPRESSION_KEY_HEADER);
    daos.writeUTF(outputCompressionStr);
    daos.close();
    headerBytes = baos.toByteArray();

}

From source file:org.apache.oozie.service.TestLiteWorkflowAppService.java

License:Apache License

public void testCreateprotoConf() throws Exception {
    Services services = new Services();
    try {// ww w  .  j  ava2 s  .c o  m
        services.init();
        Reader reader = IOUtils.getResourceAsReader("wf-schema-valid.xml", -1);
        Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
        IOUtils.copyCharStream(reader, writer);

        createTestCaseSubDir("lib");
        writer = new FileWriter(new File(getTestCaseDir(), "lib/maputil.jar"));
        writer.write("bla bla");
        writer.close();
        writer = new FileWriter(new File(getTestCaseDir(), "lib/reduceutil.so"));
        writer.write("bla bla");
        writer.close();
        createTestCaseSubDir("scripts");
        writer = new FileWriter(new File(getTestCaseDir(), "scripts/myscript.sh"));
        writer.write("bla bla");
        writer.close();
        WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
        Configuration jobConf = new XConfiguration();
        jobConf.set(OozieClient.APP_PATH, getTestCaseFileUri("workflow.xml"));
        jobConf.set(OozieClient.USER_NAME, getTestUser());

        Configuration protoConf = wps.createProtoActionConf(jobConf, true);
        assertEquals(getTestUser(), protoConf.get(OozieClient.USER_NAME));

        assertEquals(2, protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST).length);
        List<String> found = new ArrayList<String>();
        found.add(protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[0]);
        found.add(protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[1]);
        List<String> expected = new ArrayList<String>();
        expected.add(getTestCaseFileUri("lib/reduceutil.so"));
        expected.add(getTestCaseFileUri("lib/maputil.jar"));

        Collections.sort(found);
        Collections.sort(expected);
        for (int i = 0; i < expected.size(); i++) {
            assert (new URI(expected.get(i)).equals(new URI(found.get(i))));
        }
        Assert.assertTrue(!new URI(found.get(0)).equals(new URI(found.get(1))));
    } finally {
        services.destroy();
    }
}

From source file:org.apache.oozie.service.TestLiteWorkflowAppService.java

License:Apache License

public void testCreateprotoConfWithLibPath() throws Exception {
    Services services = new Services();
    try {/*from ww w  . j  a  va  2 s  .c o  m*/
        services.init();
        Reader reader = IOUtils.getResourceAsReader("wf-schema-valid.xml", -1);
        Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
        IOUtils.copyCharStream(reader, writer);

        createTestCaseSubDir("lib");
        writer = new FileWriter(new File(getTestCaseDir(), "lib/maputil.jar"));
        writer.write("bla bla");
        writer.close();
        writer = new FileWriter(new File(getTestCaseDir(), "lib/reduceutil.so"));
        writer.write("bla bla");
        writer.close();
        createTestCaseSubDir("libx");
        writer = new FileWriter(new File(getTestCaseDir(), "libx/maputilx.jar"));
        writer.write("bla bla");
        writer.close();
        WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
        Configuration jobConf = new XConfiguration();
        jobConf.set(OozieClient.APP_PATH, getTestCaseFileUri("workflow.xml"));
        jobConf.set(OozieClient.LIBPATH, getTestCaseFileUri("libx"));
        jobConf.set(OozieClient.USER_NAME, getTestUser());

        Configuration protoConf = wps.createProtoActionConf(jobConf, true);
        assertEquals(getTestUser(), protoConf.get(OozieClient.USER_NAME));

        assertEquals(3, protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST).length);
        List<String> found = new ArrayList<String>();
        found.add(protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[0]);
        found.add(protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[1]);
        found.add(protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[2]);
        List<String> expected = new ArrayList<String>();
        expected.add(getTestCaseFileUri("lib/reduceutil.so"));
        expected.add(getTestCaseFileUri("lib/maputil.jar"));
        expected.add(getTestCaseFileUri("libx/maputilx.jar"));
        Collections.sort(found);
        Collections.sort(expected);
        assertEquals(expected.size(), found.size());
        for (int i = 0; i < expected.size(); i++) {
            assert (new URI(expected.get(i)).equals(new URI(found.get(i))));
        }
    } finally {
        services.destroy();
    }
}

From source file:org.apache.oozie.service.TestLiteWorkflowAppService.java

License:Apache License

public void testCreateprotoConfWithMulipleLibPath() throws Exception {
    Services services = new Services();
    try {/*from w w w . j  a va2 s. co m*/
        services.init();
        Reader reader = IOUtils.getResourceAsReader("wf-schema-valid.xml", -1);
        Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
        IOUtils.copyCharStream(reader, writer);

        createTestCaseSubDir("lib");
        writer = new FileWriter(new File(getTestCaseDir(), "lib/maputil.jar"));
        writer.write("bla bla");
        writer.close();
        writer = new FileWriter(new File(getTestCaseDir(), "lib/reduceutil.so"));
        writer.write("bla bla");
        writer.close();
        createTestCaseSubDir("libx");
        writer = new FileWriter(new File(getTestCaseDir(), "libx/maputil_x.jar"));
        writer.write("bla bla");
        writer.close();
        createTestCaseSubDir("liby");
        writer = new FileWriter(new File(getTestCaseDir(), "liby/maputil_y1.jar"));
        writer.write("bla bla");
        writer.close();
        writer = new FileWriter(new File(getTestCaseDir(), "liby/maputil_y2.jar"));
        writer.write("bla bla");
        writer.close();
        createTestCaseSubDir("libz");
        writer = new FileWriter(new File(getTestCaseDir(), "libz/maputil_z.jar"));
        writer.write("bla bla");
        writer.close();

        WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
        Configuration jobConf = new XConfiguration();
        jobConf.set(OozieClient.APP_PATH, getTestCaseFileUri("workflow.xml"));
        jobConf.setStrings(OozieClient.LIBPATH, getTestCaseFileUri("libx"), getTestCaseFileUri("liby"),
                getTestCaseFileUri("libz"));
        jobConf.set(OozieClient.USER_NAME, getTestUser());

        Configuration protoConf = wps.createProtoActionConf(jobConf, true);
        assertEquals(getTestUser(), protoConf.get(OozieClient.USER_NAME));

        assertEquals(6, protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST).length);
        List<String> found = new ArrayList<String>();
        found.add(protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[0]);
        found.add(protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[1]);
        found.add(protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[2]);
        found.add(protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[3]);
        found.add(protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[4]);
        found.add(protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[5]);
        List<String> expected = new ArrayList<String>();
        expected.add(getTestCaseFileUri("lib/reduceutil.so"));
        expected.add(getTestCaseFileUri("lib/maputil.jar"));
        expected.add(getTestCaseFileUri("libx/maputil_x.jar"));
        expected.add(getTestCaseFileUri("liby/maputil_y1.jar"));
        expected.add(getTestCaseFileUri("liby/maputil_y2.jar"));
        expected.add(getTestCaseFileUri("libz/maputil_z.jar"));
        Collections.sort(expected);
        Collections.sort(found);
        assertEquals(expected.size(), found.size());
        for (int i = 0; i < expected.size(); i++) {
            assert (new URI(expected.get(i)).equals(new URI(found.get(i))));
        }
    } finally {
        services.destroy();
    }
}