Example usage for org.apache.hadoop.conf Configuration getConfResourceAsReader

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getConfResourceAsReader.

Prototype

public Reader getConfResourceAsReader(String name) 

Source Link

Document

Get a Reader attached to the configuration resource with the given name.

Usage

From source file:org.apache.nutch.urlfilter.automaton.AutomatonURLFilter.java

License:Apache License

/**
 * Rules specified as a config property will override rules specified as a
 * config file./*from w w  w .j  a v a2 s . co  m*/
 */
protected Reader getRulesReader(Configuration conf) throws IOException {
    String stringRules = conf.get(URLFILTER_AUTOMATON_RULES);
    if (stringRules != null) {
        return new StringReader(stringRules);
    }
    String fileRules = conf.get(URLFILTER_AUTOMATON_FILE);
    return conf.getConfResourceAsReader(fileRules);
}

---

From source file:org.apache.nutch.urlfilter.domain.DomainURLFilter.java

License:Apache License

/**
 * Sets the configuration./*from  ww  w. j a  v a2  s .  co m*/
 */
public void setConf(Configuration conf) {
    this.conf = conf;

    // get the extensions for domain urlfilter
    String pluginName = "urlfilter-domain";
    Extension[] extensions = PluginRepository.get(conf).getExtensionPoint(URLFilter.class.getName())
            .getExtensions();
    for (int i = 0; i < extensions.length; i++) {
        Extension extension = extensions[i];
        if (extension.getDescriptor().getPluginId().equals(pluginName)) {
            attributeFile = extension.getAttribute("file");
            break;
        }
    }

    // handle blank non empty input
    if (attributeFile != null && attributeFile.trim().equals("")) {
        attributeFile = null;
    }

    if (attributeFile != null) {
        if (LOG.isInfoEnabled()) {
            LOG.info("Attribute \"file\" is defined for plugin " + pluginName + " as " + attributeFile);
        }
    } else {
        if (LOG.isWarnEnabled()) {
            LOG.warn("Attribute \"file\" is not defined in plugin.xml for plugin " + pluginName);
        }
    }

    // domain file and attribute "file" take precedence if defined
    String file = conf.get("urlfilter.domain.file");
    String stringRules = conf.get("urlfilter.domain.rules");
    if (domainFile != null) {
        file = domainFile;
    } else if (attributeFile != null) {
        file = attributeFile;
    }
    Reader reader = null;
    if (stringRules != null) { // takes precedence over files
        reader = new StringReader(stringRules);
    } else {
        reader = conf.getConfResourceAsReader(file);
    }
    try {
        if (reader == null) {
            reader = new FileReader(file);
        }
        readConfiguration(reader);
    } catch (IOException e) {
        LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
    }
}

---

From source file:org.apache.nutch.urlfilter.domainblacklist.DomainBlacklistURLFilter.java

License:Apache License

/**
 * Sets the configuration.//from ww w  .jav a  2  s .  co m
 */
public void setConf(Configuration conf) {
    this.conf = conf;

    // get the extensions for domain urlfilter
    String pluginName = "urlfilter-domainblacklist";
    Extension[] extensions = PluginRepository.get(conf).getExtensionPoint(URLFilter.class.getName())
            .getExtensions();
    for (int i = 0; i < extensions.length; i++) {
        Extension extension = extensions[i];
        if (extension.getDescriptor().getPluginId().equals(pluginName)) {
            attributeFile = extension.getAttribute("file");
            break;
        }
    }

    // handle blank non empty input
    if (attributeFile != null && attributeFile.trim().equals("")) {
        attributeFile = null;
    }

    if (attributeFile != null) {
        if (LOG.isInfoEnabled()) {
            LOG.info("Attribute \"file\" is defined for plugin " + pluginName + " as " + attributeFile);
        }
    } else {
        if (LOG.isWarnEnabled()) {
            LOG.warn("Attribute \"file\" is not defined in plugin.xml for plugin " + pluginName);
        }
    }

    // domain file and attribute "file" take precedence if defined
    String file = conf.get("urlfilter.domainblacklist.file");
    String stringRules = conf.get("urlfilter.domainblacklist.rules");
    if (domainFile != null) {
        file = domainFile;
    } else if (attributeFile != null) {
        file = attributeFile;
    }
    Reader reader = null;
    if (stringRules != null) { // takes precedence over files
        reader = new StringReader(stringRules);
    } else {
        reader = conf.getConfResourceAsReader(file);
    }
    try {
        if (reader == null) {
            reader = new FileReader(file);
        }
        readConfiguration(reader);
    } catch (IOException e) {
        LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
    }
}

---

From source file:org.apache.nutch.urlfilter.exactduplicate.ExactDuplicateURLFilter.java

License:Apache License

public void setConf(Configuration conf) {
    this.conf = conf;

    String pluginName = "urlfilter-suffix";
    Extension[] extensions = PluginRepository.get(conf).getExtensionPoint(URLFilter.class.getName())
            .getExtensions();/*  w  w w .  j a  v  a 2  s.c  o  m*/
    for (int i = 0; i < extensions.length; i++) {
        Extension extension = extensions[i];
        if (extension.getDescriptor().getPluginId().equals(pluginName)) {
            attributeFile = extension.getAttribute("file");
            break;
        }
    }
    if (attributeFile != null && attributeFile.trim().equals(""))
        attributeFile = null;
    if (attributeFile != null) {
        if (LOG.isInfoEnabled()) {
            LOG.info("Attribute \"file\" is defined for plugin " + pluginName + " as " + attributeFile);
        }
    } else {
        // if (LOG.isWarnEnabled()) {
        // LOG.warn("Attribute \"file\" is not defined in plugin.xml for
        // plugin "+pluginName);
        // }
    }

    String file = conf.get("urlfilter.suffix.file");
    String stringRules = conf.get("urlfilter.suffix.rules");
    // attribute "file" takes precedence if defined
    if (attributeFile != null)
        file = attributeFile;
    Reader reader = null;
    if (stringRules != null) { // takes precedence over files
        reader = new StringReader(stringRules);
    } else {
        reader = conf.getConfResourceAsReader(file);
    }

    /*try {
    //readConfiguration(reader);
    } catch (IOException e) {
    if (LOG.isErrorEnabled()) {
    LOG.error(e.getMessage());
    }
    throw new RuntimeException(e.getMessage(), e);
    }*/
}

---

From source file:org.apache.nutch.urlfilter.forbidden.ForbiddenURLFilter.java

License:Apache License

public void setConf(Configuration conf) {
    this.conf = conf;

    String pluginName = "urlfilter-forbidden";
    Extension[] extensions = PluginRepository.get(conf).getExtensionPoint(URLFilter.class.getName())
            .getExtensions();//from w  ww . ja  v a  2s .co  m
    for (int i = 0; i < extensions.length; i++) {
        Extension extension = extensions[i];
        if (extension.getDescriptor().getPluginId().equals(pluginName)) {
            attributeFile = extension.getAttribute("file");
            break;
        }
    }
    if (attributeFile != null && attributeFile.trim().equals(""))
        attributeFile = null;
    if (attributeFile != null) {
        if (LOG.isInfoEnabled()) {
            LOG.info("Attribute \"file\" is defined for plugin " + pluginName + " as " + attributeFile);
        }
    } else {
        // if (LOG.isWarnEnabled()) {
        // LOG.warn("Attribute \"file\" is not defined in plugin.xml for
        // plugin "+pluginName);
        // }
    }

    String file = conf.get("urlfilter.forbidden.file");

    // attribute "file" takes precedence if defined
    if (attributeFile != null)
        file = attributeFile;
    Reader reader = conf.getConfResourceAsReader(file);

    try {
        readConfiguration(reader);
    } catch (IOException e) {
        if (LOG.isErrorEnabled()) {
            LOG.error(e.getMessage());
        }
        throw new RuntimeException(e.getMessage(), e);
    }
}

---

From source file:org.apache.nutch.urlfilter.ignoreexempt.ExemptionUrlFilter.java

License:Apache License

/**
 * Gets reader for regex rules/*from  w ww. j  ava 2  s. c om*/
 */
protected Reader getRulesReader(Configuration conf) throws IOException {
    String fileRules = conf.get(DB_IGNORE_EXTERNAL_EXEMPTIONS_FILE);
    return conf.getConfResourceAsReader(fileRules);
}

---

From source file:org.apache.nutch.urlfilter.prefix.PrefixURLFilter.java

License:Apache License

public void setConf(Configuration conf) {
    this.conf = conf;

    String pluginName = "urlfilter-prefix";
    Extension[] extensions = PluginRepository.get(conf).getExtensionPoint(URLFilter.class.getName())
            .getExtensions();/*from   ww  w .  ja  va  2  s.com*/
    for (int i = 0; i < extensions.length; i++) {
        Extension extension = extensions[i];
        if (extension.getDescriptor().getPluginId().equals(pluginName)) {
            attributeFile = extension.getAttribute("file");
            break;
        }
    }
    if (attributeFile != null && attributeFile.trim().equals(""))
        attributeFile = null;
    if (attributeFile != null) {
        if (LOG.isInfoEnabled()) {
            LOG.info("Attribute \"file\" is defined for plugin " + pluginName + " as " + attributeFile);
        }
    } else {
        // if (LOG.isWarnEnabled()) {
        // LOG.warn("Attribute \"file\" is not defined in plugin.xml for
        // plugin "+pluginName);
        // }
    }

    String file = conf.get("urlfilter.prefix.file");
    String stringRules = conf.get("urlfilter.prefix.rules");
    // attribute "file" takes precedence if defined
    if (attributeFile != null)
        file = attributeFile;
    Reader reader = null;
    if (stringRules != null) { // takes precedence over files
        reader = new StringReader(stringRules);
    } else {
        reader = conf.getConfResourceAsReader(file);
    }

    if (reader == null) {
        trie = new PrefixStringMatcher(new String[0]);
    } else {
        try {
            trie = readConfiguration(reader);
        } catch (IOException e) {
            if (LOG.isErrorEnabled()) {
                LOG.error(e.getMessage());
            }
            // TODO mb@media-style.com: throw Exception? Because broken api.
            throw new RuntimeException(e.getMessage(), e);
        }
    }
}

---

From source file:org.apache.nutch.urlfilter.suffix.SuffixURLFilter.java

License:Apache License

public void setConf(Configuration conf) {
    this.conf = conf;

    String pluginName = "urlfilter-suffix";
    Extension[] extensions = PluginRepository.get(conf).getExtensionPoint(URLFilter.class.getName())
            .getExtensions();//from w  ww  .j a  v a  2s  .c o  m
    for (int i = 0; i < extensions.length; i++) {
        Extension extension = extensions[i];
        if (extension.getDescriptor().getPluginId().equals(pluginName)) {
            attributeFile = extension.getAttribute("file");
            break;
        }
    }
    if (attributeFile != null && attributeFile.trim().equals(""))
        attributeFile = null;
    if (attributeFile != null) {
        if (LOG.isInfoEnabled()) {
            LOG.info("Attribute \"file\" is defined for plugin " + pluginName + " as " + attributeFile);
        }
    } else {
        // if (LOG.isWarnEnabled()) {
        // LOG.warn("Attribute \"file\" is not defined in plugin.xml for
        // plugin "+pluginName);
        // }
    }

    String file = conf.get("urlfilter.suffix.file");
    String stringRules = conf.get("urlfilter.suffix.rules");
    // attribute "file" takes precedence if defined
    if (attributeFile != null)
        file = attributeFile;
    Reader reader = null;
    if (stringRules != null) { // takes precedence over files
        reader = new StringReader(stringRules);
    } else {
        reader = conf.getConfResourceAsReader(file);
    }

    try {
        readConfiguration(reader);
    } catch (IOException e) {
        if (LOG.isErrorEnabled()) {
            LOG.error(e.getMessage());
        }
        throw new RuntimeException(e.getMessage(), e);
    }
}

---

• «
• 1
• 2
• 3
• 4
• »