Example usage for org.apache.hadoop.conf Configuration set

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration set.

Prototype

public void set(String name, String value)

Source Link

Document

Set the value of the name property.

Usage

From source file:CalculateSentiment.WordCount.java

License:Apache License

public static void main(String[] args) throws Exception {
    Path tempDir = new Path("wordcount-temp-" + Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));

    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 3) {
        System.err.println("Usage: wordcount <in> <out> <category>");
        System.exit(2);/*  w w  w. j  a  v a  2 s .c  o  m*/
    }
    conf.set("category", otherArgs[2]);

    // try {
    //     String filePath = otherArgs[0];
    //     BufferedReader br = new BufferedReader(new FileReader(filePath));
    //     String line = br.readLine();
    //     conf.set("category", line);
    // } catch (Exception e) {
    //     e.printStackTrace();
    // }
    // conf.set("category", WordCount.read(otherArgs[2]));

    DistributedCache.createSymlink(conf);
    String path = "CalculateSentiment.obj";
    Path filePath = new Path(path);
    String uriWithLink = filePath.toUri().toString() + "#" + "object";
    DistributedCache.addCacheFile(new URI(uriWithLink), conf);

    // DistributedCache.addCacheFile(new URI("/CalculateSentiment.obj"), conf);
    Job job = new Job(conf, "Test");

    job.setJarByClass(WordCount.class);
    job.setMapperClass(TokenizerMapper.class);
    job.setCombinerClass(DoubleSumReducer.class);
    job.setReducerClass(DoubleSumReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    job.setNumReduceTasks(1);
    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

From source file:cascading.flow.hadoop.FlowPlatformTest.java

License:Open Source License

@Test
public void testCopyConfig() throws Exception {
    Tap source = new Lfs(new TextLine(), "input/path");
    Tap sink = new Hfs(new TextLine(), "output/path", SinkMode.REPLACE);

    Pipe pipe = new Pipe("test");

    Configuration conf = ((BaseHadoopPlatform) getPlatform()).getConfiguration();

    conf.set(AppProps.APP_NAME, "testname");

    AppProps props = AppProps.appProps().setVersion("1.2.3");

    Properties properties = props.buildProperties(conf); // convert job conf to properties instance

    Flow flow = getPlatform().getFlowConnector(properties).connect(source, sink, pipe);

    assertEquals("testname", flow.getProperty(AppProps.APP_NAME));
    assertEquals("1.2.3", flow.getProperty(AppProps.APP_VERSION));
}

From source file:cascading.flow.hadoop.util.HadoopUtil.java

License:Open Source License

public static <J extends Configuration> J mergeConf(J job, Map<String, String> config, boolean directly) {
    Configuration currentConf = directly ? job
            : (job instanceof JobConf ? copyJobConf((JobConf) job) : new Configuration(job));

    for (String key : config.keySet()) {
        LOG.debug("merging key: {} value: {}", key, config.get(key));

        currentConf.set(key, config.get(key));
    }/*  www .ja va  2s  .c o m*/

    return (J) currentConf;
}

From source file:cascading.flow.hadoop.util.HadoopUtil.java

License:Open Source License

public static void setLocal(Configuration conf) {
    // set both properties to local
    conf.set("mapred.job.tracker", "local");

    // yarn/*from  w w w. j av  a2 s  .co m*/
    conf.set("mapreduce.framework.name", "local");

    // tez
    conf.set("tez.local.mode", "true");
    conf.set("tez.runtime.optimize.local.fetch", "true");
}

From source file:cascading.flow.hadoop.util.HadoopUtil.java

License:Open Source License

public static void addInputPath(Configuration conf, Path path) {
    Path workingDirectory = getWorkingDirectory(conf);
    path = new Path(workingDirectory, path);
    String dirStr = StringUtils.escapeString(path.toString());
    String dirs = conf.get("mapred.input.dir");
    conf.set("mapred.input.dir", dirs == null ? dirStr : dirs + StringUtils.COMMA_STR + dirStr);
}

From source file:cascading.flow.hadoop.util.HadoopUtil.java

License:Open Source License

public static void setOutputPath(Configuration conf, Path path) {
    Path workingDirectory = getWorkingDirectory(conf);
    path = new Path(workingDirectory, path);
    conf.set("mapred.output.dir", path.toString());
}

From source file:cascading.flow.hadoop.util.HadoopUtil.java

License:Open Source License

private static Path getWorkingDirectory(Configuration conf) {
    String name = conf.get("mapred.working.dir");
    if (name != null) {
        return new Path(name);
    } else {/* ww w  .  j av  a2  s .  c om*/
        try {
            Path dir = FileSystem.get(conf).getWorkingDirectory();
            conf.set("mapred.working.dir", dir.toString());
            return dir;
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
}

From source file:cascading.flow.hadoop.util.HadoopUtil.java

License:Open Source License

public static void addComparators(Configuration conf, String property, Map<String, Fields> map,
        BaseFlowStep flowStep, Group group) {
    Iterator<Fields> fieldsIterator = map.values().iterator();

    if (!fieldsIterator.hasNext())
        return;/* w w w.  j a v a 2s.  c  o  m*/

    Fields fields = fieldsIterator.next();

    if (fields.hasComparators()) {
        conf.set(property, pack(fields, conf));
        return;
    }

    // use resolved fields if there are no comparators.
    Set<Scope> previousScopes = flowStep.getPreviousScopes(group);

    fields = previousScopes.iterator().next().getOutValuesFields();

    if (fields.size() != 0) // allows fields.UNKNOWN to be used
        conf.setInt(property + ".size", fields.size());
}

From source file:cascading.flow.hadoop.UtilHadoopTest.java

License:Open Source License

@Test
public void testSetLogLevel() {
    Configuration jobConf = new Configuration();

    jobConf.set("log4j.logger", "cascading=DEBUG");

    HadoopUtil.initLog4j(jobConf);// w  w w. j a  v  a  2s  .  c  o  m

    Object loggerObject = Util.invokeStaticMethod("org.apache.log4j.Logger", "getLogger",
            new Object[] { "cascading" }, new Class[] { String.class });

    Object levelObject = Util.invokeStaticMethod("org.apache.log4j.Level", "toLevel", new Object[] { "DEBUG" },
            new Class[] { String.class });

    Object returnedLevel = Util.invokeInstanceMethod(loggerObject, "getLevel", new Object[] {}, new Class[] {});

    assertEquals(levelObject, returnedLevel);
}

From source file:cascading.flow.hadoop2.Hadoop2MR1Planner.java

License:Open Source License

/**
 * Method copyProperties adds the given Map values to the given JobConf object.
 *
 * @param configuration of type JobConf//from   w w w.  ja  va2  s .  co m
 * @param properties    of type Map
 */
public static void copyProperties(Configuration configuration, Map<Object, Object> properties) {
    if (properties instanceof Properties) {
        Properties props = (Properties) properties;
        Set<String> keys = props.stringPropertyNames();

        for (String key : keys)
            configuration.set(key, props.getProperty(key));
    } else {
        for (Map.Entry<Object, Object> entry : properties.entrySet()) {
            if (entry.getValue() != null)
                configuration.set(entry.getKey().toString(), entry.getValue().toString());
        }
    }
}