Example usage for org.apache.hadoop.conf Configuration setStrings

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setStrings.

Prototype

public void setStrings(String name, String... values) 

Source Link

Document

Set the array of string values for the name property as as comma delimited values.

Usage

From source file:org.apache.ignite.internal.processors.hadoop.GridHadoopTaskExecutionSelfTest.java

License:Apache License

/**
 * @throws Exception If failed.//w w w  .j a  va 2s  . c om
 */
public void testMapRun() throws Exception {
    int lineCnt = 10000;
    String fileName = "/testFile";

    prepareFile(fileName, lineCnt);

    totalLineCnt.set(0);
    taskWorkDirs.clear();

    Configuration cfg = new Configuration();

    cfg.setStrings("fs.igfs.impl", IgfsHadoopFileSystem.class.getName());

    Job job = Job.getInstance(cfg);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setMapperClass(TestMapper.class);

    job.setNumReduceTasks(0);

    job.setInputFormatClass(TextInputFormat.class);

    FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/"));
    FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output/"));

    job.setJarByClass(getClass());

    IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new GridHadoopJobId(UUID.randomUUID(), 1),
            createJobInfo(job.getConfiguration()));

    fut.get();

    assertEquals(lineCnt, totalLineCnt.get());

    assertEquals(32, taskWorkDirs.size());
}

---

From source file:org.apache.ignite.internal.processors.hadoop.GridHadoopTaskExecutionSelfTest.java

License:Apache License

/**
 * @throws Exception If failed.//from  w w  w  . java 2s .  c  o  m
 */
public void testMapCombineRun() throws Exception {
    int lineCnt = 10001;
    String fileName = "/testFile";

    prepareFile(fileName, lineCnt);

    totalLineCnt.set(0);
    taskWorkDirs.clear();

    Configuration cfg = new Configuration();

    cfg.setStrings("fs.igfs.impl", IgfsHadoopFileSystem.class.getName());
    cfg.setBoolean(MAP_WRITE, true);

    Job job = Job.getInstance(cfg);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setMapperClass(TestMapper.class);
    job.setCombinerClass(TestCombiner.class);
    job.setReducerClass(TestReducer.class);

    job.setNumReduceTasks(2);

    job.setInputFormatClass(TextInputFormat.class);

    FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/"));
    FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output"));

    job.setJarByClass(getClass());

    GridHadoopJobId jobId = new GridHadoopJobId(UUID.randomUUID(), 2);

    IgniteInternalFuture<?> fut = grid(0).hadoop().submit(jobId, createJobInfo(job.getConfiguration()));

    fut.get();

    assertEquals(lineCnt, totalLineCnt.get());

    assertEquals(34, taskWorkDirs.size());

    for (int g = 0; g < gridCount(); g++)
        grid(g).hadoop().finishFuture(jobId).get();
}

---

From source file:org.apache.ignite.internal.processors.hadoop.GridHadoopTaskExecutionSelfTest.java

License:Apache License

/**
 * @throws Exception If failed.//  w ww.  jav  a  2  s  .  c o  m
 */
public void testMapperException() throws Exception {
    prepareFile("/testFile", 1000);

    Configuration cfg = new Configuration();

    cfg.setStrings("fs.igfs.impl", IgfsHadoopFileSystem.class.getName());

    Job job = Job.getInstance(cfg);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setMapperClass(FailMapper.class);

    job.setNumReduceTasks(0);

    job.setInputFormatClass(TextInputFormat.class);

    FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/"));
    FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output/"));

    job.setJarByClass(getClass());

    final IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new GridHadoopJobId(UUID.randomUUID(), 3),
            createJobInfo(job.getConfiguration()));

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override
        public Object call() throws Exception {
            fut.get();

            return null;
        }
    }, IgniteCheckedException.class, null);
}

---

From source file:org.apache.ignite.internal.processors.hadoop.HadoopTaskExecutionSelfTest.java

License:Apache License

/**
 * @throws Exception If failed./*  w ww .j a va 2  s.  co m*/
 */
public void testMapRun() throws Exception {
    int lineCnt = 10000;
    String fileName = "/testFile";

    prepareFile(fileName, lineCnt);

    totalLineCnt.set(0);
    taskWorkDirs.clear();

    Configuration cfg = new Configuration();

    cfg.setStrings("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());

    Job job = Job.getInstance(cfg);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setMapperClass(TestMapper.class);

    job.setNumReduceTasks(0);

    job.setInputFormatClass(TextInputFormat.class);

    FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/"));
    FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output/"));

    job.setJarByClass(getClass());

    IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 1),
            createJobInfo(job.getConfiguration()));

    fut.get();

    assertEquals(lineCnt, totalLineCnt.get());

    assertEquals(32, taskWorkDirs.size());
}

---

From source file:org.apache.ignite.internal.processors.hadoop.HadoopTaskExecutionSelfTest.java

License:Apache License

/**
 * @throws Exception If failed./*from  w ww. j  a  va  2s .c  om*/
 */
public void testMapCombineRun() throws Exception {
    int lineCnt = 10001;
    String fileName = "/testFile";

    prepareFile(fileName, lineCnt);

    totalLineCnt.set(0);
    taskWorkDirs.clear();

    Configuration cfg = new Configuration();

    cfg.setStrings("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());
    cfg.setBoolean(MAP_WRITE, true);

    Job job = Job.getInstance(cfg);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setMapperClass(TestMapper.class);
    job.setCombinerClass(TestCombiner.class);
    job.setReducerClass(TestReducer.class);

    job.setNumReduceTasks(2);

    job.setInputFormatClass(TextInputFormat.class);

    FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/"));
    FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output"));

    job.setJarByClass(getClass());

    HadoopJobId jobId = new HadoopJobId(UUID.randomUUID(), 2);

    IgniteInternalFuture<?> fut = grid(0).hadoop().submit(jobId, createJobInfo(job.getConfiguration()));

    fut.get();

    assertEquals(lineCnt, totalLineCnt.get());

    assertEquals(34, taskWorkDirs.size());

    for (int g = 0; g < gridCount(); g++)
        grid(g).hadoop().finishFuture(jobId).get();
}

---

From source file:org.apache.ignite.internal.processors.hadoop.HadoopTaskExecutionSelfTest.java

License:Apache License

/**
 * @throws Exception If failed.//from w w w  .  j  a v a 2  s. com
 */
public void testMapperException() throws Exception {
    prepareFile("/testFile", 1000);

    Configuration cfg = new Configuration();

    cfg.setStrings("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());

    Job job = Job.getInstance(cfg);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setMapperClass(FailMapper.class);

    job.setNumReduceTasks(0);

    job.setInputFormatClass(TextInputFormat.class);

    FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/"));
    FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output/"));

    job.setJarByClass(getClass());

    final IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 3),
            createJobInfo(job.getConfiguration()));

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override
        public Object call() throws Exception {
            fut.get();

            return null;
        }
    }, IgniteCheckedException.class, null);
}

---

From source file:org.apache.ignite.internal.processors.hadoop.impl.HadoopTaskExecutionSelfTest.java

License:Apache License

/**
 * @throws Exception If failed.//  www  .  j av  a2  s  . co m
 */
public void testMapRun() throws Exception {
    int lineCnt = 10000;
    String fileName = "/testFile";

    prepareFile(fileName, lineCnt);

    totalLineCnt.set(0);
    taskWorkDirs.clear();

    Configuration cfg = new Configuration();

    cfg.setStrings("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());

    Job job = Job.getInstance(cfg);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setMapperClass(TestMapper.class);

    job.setNumReduceTasks(0);

    job.setInputFormatClass(TextInputFormat.class);

    FileInputFormat.setInputPaths(job, new Path("igfs://" + igfsName + "@/"));
    FileOutputFormat.setOutputPath(job, new Path("igfs://" + igfsName + "@/output/"));

    job.setJarByClass(getClass());

    IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 1),
            createJobInfo(job.getConfiguration()));

    fut.get();

    assertEquals(lineCnt, totalLineCnt.get());

    assertEquals(32, taskWorkDirs.size());
}

---

From source file:org.apache.ignite.internal.processors.hadoop.impl.HadoopTaskExecutionSelfTest.java

License:Apache License

/**
 * @throws Exception If failed./*  w w w.ja v a2s  .c  om*/
 */
public void testMapCombineRun() throws Exception {
    int lineCnt = 10001;
    String fileName = "/testFile";

    prepareFile(fileName, lineCnt);

    totalLineCnt.set(0);
    taskWorkDirs.clear();

    Configuration cfg = new Configuration();

    cfg.setStrings("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());
    cfg.setBoolean(MAP_WRITE, true);

    Job job = Job.getInstance(cfg);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setMapperClass(TestMapper.class);
    job.setCombinerClass(TestCombiner.class);
    job.setReducerClass(TestReducer.class);

    job.setNumReduceTasks(2);

    job.setInputFormatClass(TextInputFormat.class);

    FileInputFormat.setInputPaths(job, new Path("igfs://" + igfsName + "@/"));
    FileOutputFormat.setOutputPath(job, new Path("igfs://" + igfsName + "@/output/"));

    job.setJarByClass(getClass());

    HadoopJobId jobId = new HadoopJobId(UUID.randomUUID(), 2);

    IgniteInternalFuture<?> fut = grid(0).hadoop().submit(jobId, createJobInfo(job.getConfiguration()));

    fut.get();

    assertEquals(lineCnt, totalLineCnt.get());

    assertEquals(34, taskWorkDirs.size());

    for (int g = 0; g < gridCount(); g++)
        grid(g).hadoop().finishFuture(jobId).get();
}

---

From source file:org.apache.ignite.internal.processors.hadoop.impl.HadoopTaskExecutionSelfTest.java

License:Apache License

/**
 * @throws Exception If failed.//  w w  w  .ja va  2  s  .  c o  m
 */
public void testMapperException() throws Exception {
    prepareFile("/testFile", 1000);

    Configuration cfg = new Configuration();

    cfg.setStrings("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());

    Job job = Job.getInstance(cfg);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setMapperClass(FailMapper.class);

    job.setNumReduceTasks(0);

    job.setInputFormatClass(TextInputFormat.class);

    FileInputFormat.setInputPaths(job, new Path("igfs://" + igfsName + "@/"));
    FileOutputFormat.setOutputPath(job, new Path("igfs://" + igfsName + "@/output/"));

    job.setJarByClass(getClass());

    final IgniteInternalFuture<?> fut = grid(0).hadoop().submit(new HadoopJobId(UUID.randomUUID(), 3),
            createJobInfo(job.getConfiguration()));

    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override
        public Object call() throws Exception {
            fut.get();

            return null;
        }
    }, IgniteCheckedException.class, null);
}

---

From source file:org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemClientBasedAbstractSelfTest.java

License:Apache License

/** {@inheritDoc} */
@Override/*w ww  .ja v  a  2 s  .  c  om*/
protected Configuration configuration(String authority, boolean skipEmbed, boolean skipLocShmem) {
    Configuration cfg = new Configuration();

    cfg.set("fs.defaultFS", "igfs://" + authority + "/");
    cfg.set("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());
    cfg.set("fs.AbstractFileSystem.igfs.impl",
            org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.class.getName());

    cfg.setBoolean("fs.igfs.impl.disable.cache", true);
    cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, authority), true);
    cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority), true);
    cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_LOCAL_TCP, authority), true);
    cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_REMOTE_TCP, authority), true);
    cfg.setStrings(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_IGNITE_CFG_PATH, authority),
            getClientConfig());

    return cfg;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• »