Example usage for org.apache.hadoop.conf Configuration setStrings

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setStrings.

Prototype

public void setStrings(String name, String... values) 

Source Link

Document

Set the array of string values for the name property as as comma delimited values.

Usage

From source file:org.hipi.tools.downloader.Downloader.java

License:Open Source License

public int run(String[] args) throws Exception {

    // try to parse command line arguments
    CommandLine line = null;// ww w. j ava2  s.co m
    try {
        line = parser.parse(options, args);
    } catch (ParseException exp) {
        usage();
    }
    if (line == null) {
        usage();
    }

    String[] leftArgs = line.getArgs();

    if (leftArgs.length != 2) {
        usage();
    }

    String inputDir = leftArgs[0];
    String outputHib = leftArgs[1];

    boolean yfcc100m = line.hasOption("yfcc100m");
    int numDownloadNodes = (yfcc100m ? 1
            : ((line.hasOption("num-nodes") ? Integer.parseInt(line.getOptionValue("num-nodes")) : 1)));
    if (numDownloadNodes < 1) {
        System.err.println("Invalid number of download nodes specified [" + numDownloadNodes + "]");
        System.exit(1);
    }

    boolean overwrite = line.hasOption("force");

    System.out.println("Source directory: " + inputDir);
    System.out.println("Output HIB: " + outputHib);
    System.out.println("Overwrite output HIB if it exists: " + (overwrite ? "true" : "false"));
    System.out.println("YFCC100M format: " + (yfcc100m ? "true" : "false"));
    System.out.println("Number of download nodes: " + numDownloadNodes);

    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);

    // Remove existing HIB if overwrite is specified and HIB exists
    if (!overwrite) {
        if (fs.exists(new Path(outputHib))) {
            System.err.println(
                    "HIB [" + outputHib + "] already exists. Use the \"--force\" argument to overwrite.");
            System.exit(1);
        }
    } else { // overwrite
        if (fs.exists(new Path(outputHib))) {
            System.out.println("Found that output HIB already exists, deleting.");
        }
    }

    fs.delete(new Path(outputHib), true);
    fs.delete(new Path(outputHib + ".dat"), true);
    fs.delete(new Path(outputHib + "_output"), true);

    // Scan source directory for list of input files
    FileStatus[] inputFiles = fs.listStatus(new Path(inputDir));
    if (inputFiles == null || inputFiles.length == 0) {
        System.err.println("Failed to find any files in source directory: " + inputDir);
        System.exit(1);
    }

    // Validate list of input files
    ArrayList<Path> sourceFiles = new ArrayList<Path>();
    for (FileStatus file : inputFiles) {

        Path path = file.getPath();

        if (yfcc100m) {
            String[] tokens = path.getName().split("-");
            if (tokens == null || tokens.length == 0) {
                System.out.println("  Skipping source file (does not follow YFCC100M file name convention): "
                        + file.getPath());
                continue;
            }
        }

        try {
            // If it exists, get the relevant compression codec
            CompressionCodecFactory codecFactory = new CompressionCodecFactory(conf);
            CompressionCodec codec = codecFactory.getCodec(path);

            FSDataInputStream fis = fs.open(path);

            // If the codec was found, use it to create an decompressed input stream.
            // Otherwise, assume input stream is already decompressed
            BufferedReader reader = null;
            if (codec != null) {
                reader = new BufferedReader(new InputStreamReader(codec.createInputStream(fis)));
            } else {
                reader = new BufferedReader(new InputStreamReader(fis));
            }

            String fileLine = reader.readLine();
            String[] lineFields = (yfcc100m ? fileLine.split("\t") : fileLine.split("\\s+"));

            if (yfcc100m) {
                if (lineFields.length != 23) {
                    System.out.println("  Skipping source file (does not follow YFCC100M source file format): "
                            + file.getPath());
                    String imageUri = null;
                } else {
                    System.out.println("  Adding source file: " + file.getPath());
                    sourceFiles.add(path);
                }
            } else {
                if (lineFields.length != 1) {
                    System.out.println(
                            "  Skipping source file (contains multiple fields per line where only one is expected): "
                                    + file.getPath());
                    if (lineFields.length == 23) {
                        System.out.println("  Did you mean to use \"--yfcc100m\"?");
                    }
                    String imageUri = null;
                } else {
                    System.out.println("  Adding source file: " + file.getPath());
                    sourceFiles.add(path);
                }
            }
            fis.close();
            reader = null;
        } catch (Exception e) {
            System.err.println("Skipping source file (unable to open and parse first line: " + file.getPath());
            continue;
        }

    }

    if (sourceFiles.size() == 0) {
        System.err.println("Failed to find any valid files in source directory: " + inputDir);
        System.exit(1);
    }

    // Construct path to directory containing outputHib
    String outputPath = outputHib.substring(0, outputHib.lastIndexOf('/') + 1);

    // Attaching job parameters to global Configuration object
    conf.setInt("downloader.nodes", numDownloadNodes);
    conf.setStrings("downloader.outfile", outputHib);
    conf.setStrings("downloader.outpath", outputPath);
    conf.setBoolean("downloader.yfcc100m", yfcc100m);

    Job job = Job.getInstance(conf, "hibDownload");
    job.setJarByClass(Downloader.class);
    job.setMapperClass(DownloaderMapper.class);
    job.setReducerClass(DownloaderReducer.class);
    job.setInputFormatClass(DownloaderInputFormat.class);
    job.setOutputKeyClass(BooleanWritable.class);
    job.setOutputValueClass(Text.class);
    job.setNumReduceTasks(1);

    FileOutputFormat.setOutputPath(job, new Path(outputHib + "_output"));

    Path[] inputPaths = new Path[sourceFiles.size()];
    inputPaths = sourceFiles.toArray(inputPaths);
    DownloaderInputFormat.setInputPaths(job, inputPaths);

    return job.waitForCompletion(true) ? 0 : 1;
}

---

From source file:org.huahinframework.core.lib.input.creator.JoinRegexValueCreatorTest.java

License:Apache License

@Test
public void testString() throws DataFormatException {
    String[] labels = { "AAA", "BBB", "CCC", "DDD" };
    String[] masterLabels = { "DDD", "NAME" };

    Map<Pattern, String[]> simpleJoinMap = new HashMap<Pattern, String[]>();
    String[] m1 = { "^d.*D$", "DdddD" };
    simpleJoinMap.put(Pattern.compile("^d.*D$"), m1);
    String[] m2 = { "^4.*IV$", "IV" };
    simpleJoinMap.put(Pattern.compile("^4.*IV$"), m2);

    Configuration conf = new Configuration();
    conf.setStrings(SimpleJob.MASTER_LABELS, masterLabels);
    conf.set(SimpleJob.JOIN_DATA_COLUMN, "DDD");
    ValueCreator valueCreator = new JoinRegexValueCreator(labels, false, "\t", false, simpleJoinMap, conf);
    Value value = new Value();

    String string1 = "a\tb\tc\tdaaaaD";
    valueCreator.create(string1, value);
    assertEquals(value.getPrimitiveValue("AAA"), "a");
    assertEquals(value.getPrimitiveValue("BBB"), "b");
    assertEquals(value.getPrimitiveValue("CCC"), "c");
    assertEquals(value.getPrimitiveValue("DDD"), "daaaaD");
    assertEquals(value.getPrimitiveValue("NAME"), "DdddD");

    value.clear();/*w w  w . j a  va 2 s.  c o m*/
    String string2 = "1\t2\t3\t41111IV";
    valueCreator.create(string2, value);
    assertEquals(value.getPrimitiveValue("AAA"), "1");
    assertEquals(value.getPrimitiveValue("BBB"), "2");
    assertEquals(value.getPrimitiveValue("CCC"), "3");
    assertEquals(value.getPrimitiveValue("DDD"), "41111IV");
    assertEquals(value.getPrimitiveValue("NAME"), "IV");
}

---

From source file:org.huahinframework.core.lib.input.creator.JoinRegexValueCreatorTest.java

License:Apache License

@Test
public void testRegex() throws DataFormatException {
    String[] labels = { "AAA", "BBB", "CCC", "DDD" };
    String[] masterLabels = { "DDD", "NAME" };

    Map<Pattern, String[]> simpleJoinMap = new HashMap<Pattern, String[]>();
    String[] m1 = { "^d.*D$", "DdddD" };
    simpleJoinMap.put(Pattern.compile("^d.*D$"), m1);
    String[] m2 = { "^4.*IV$", "IV" };
    simpleJoinMap.put(Pattern.compile("^4.*IV$"), m2);

    Configuration conf = new Configuration();
    conf.setStrings(SimpleJob.MASTER_LABELS, masterLabels);
    conf.set(SimpleJob.JOIN_DATA_COLUMN, "DDD");
    ValueCreator valueCreator = new JoinRegexValueCreator(labels, false, "^(.*)\\t(.*)\\t(.*)\\t(.*)$", true,
            simpleJoinMap, conf);/*  ww w .j a  va 2  s  . co m*/
    Value value = new Value();

    String string1 = "a\tb\tc\tdaaaaD";
    valueCreator.create(string1, value);
    assertEquals(value.getPrimitiveValue("AAA"), "a");
    assertEquals(value.getPrimitiveValue("BBB"), "b");
    assertEquals(value.getPrimitiveValue("CCC"), "c");
    assertEquals(value.getPrimitiveValue("DDD"), "daaaaD");
    assertEquals(value.getPrimitiveValue("NAME"), "DdddD");

    value.clear();
    String string2 = "1\t2\t3\t41111IV";
    valueCreator.create(string2, value);
    assertEquals(value.getPrimitiveValue("AAA"), "1");
    assertEquals(value.getPrimitiveValue("BBB"), "2");
    assertEquals(value.getPrimitiveValue("CCC"), "3");
    assertEquals(value.getPrimitiveValue("DDD"), "41111IV");
    assertEquals(value.getPrimitiveValue("NAME"), "IV");
}

---

From source file:org.huahinframework.core.lib.input.creator.JoinSomeRegexValueCreatorTest.java

License:Apache License

@Test
public void testString() throws DataFormatException {
    String[] labels = { "AAA", "BBB", "CCC", "DDD", "EEE" };
    String[] masterLabels = { "DDD", "EEE", "NAME" };

    Map<List<Pattern>, String[]> simpleJoinMap = new HashMap<List<Pattern>, String[]>();
    String[] m1 = { "^d.*D$", "^e.*E$", "DdddD" };
    simpleJoinMap.put(Arrays.asList(Pattern.compile("^d.*D$"), Pattern.compile("^e.*E$")), m1);

    String[] m2 = { "^4.*IV$", "^5.*V$", "IV" };
    simpleJoinMap.put(Arrays.asList(Pattern.compile("^4.*IV$"), Pattern.compile("^5.*V$")), m2);

    Configuration conf = new Configuration();
    conf.setStrings(SimpleJob.MASTER_LABELS, masterLabels);
    String[] join = { "DDD", "EEE" };
    conf.setStrings(SimpleJob.JOIN_DATA_COLUMN, join);
    ValueCreator valueCreator = new JoinSomeRegexValueCreator(labels, true, "\t", false, simpleJoinMap, conf);
    Value value = new Value();

    String string1 = "a\tb\tc\tdaaaaD\teaaaaE";
    valueCreator.create(string1, value);
    assertEquals(value.getPrimitiveValue("AAA"), "a");
    assertEquals(value.getPrimitiveValue("BBB"), "b");
    assertEquals(value.getPrimitiveValue("CCC"), "c");
    assertEquals(value.getPrimitiveValue("DDD"), "daaaaD");
    assertEquals(value.getPrimitiveValue("EEE"), "eaaaaE");
    assertEquals(value.getPrimitiveValue("NAME"), "DdddD");

    value.clear();//from  w w w.j  a  v a2 s  . co m
    String string2 = "1\t2\t3\t41111IV\t51111V";
    valueCreator.create(string2, value);
    assertEquals(value.getPrimitiveValue("AAA"), "1");
    assertEquals(value.getPrimitiveValue("BBB"), "2");
    assertEquals(value.getPrimitiveValue("CCC"), "3");
    assertEquals(value.getPrimitiveValue("DDD"), "41111IV");
    assertEquals(value.getPrimitiveValue("EEE"), "51111V");
    assertEquals(value.getPrimitiveValue("NAME"), "IV");
}

---

From source file:org.huahinframework.core.lib.input.creator.JoinSomeRegexValueCreatorTest.java

License:Apache License

@Test
public void testRegex() throws DataFormatException {
    String[] labels = { "AAA", "BBB", "CCC", "DDD", "EEE" };
    String[] masterLabels = { "DDD", "EEE", "NAME" };

    Map<List<Pattern>, String[]> simpleJoinMap = new HashMap<List<Pattern>, String[]>();
    String[] m1 = { "^d.*D$", "^e.*E$", "DdddD" };
    simpleJoinMap.put(Arrays.asList(Pattern.compile("^d.*D$"), Pattern.compile("^e.*E$")), m1);

    String[] m2 = { "^4.*IV$", "^5.*V$", "IV" };
    simpleJoinMap.put(Arrays.asList(Pattern.compile("^4.*IV$"), Pattern.compile("^5.*V$")), m2);

    Configuration conf = new Configuration();
    conf.setStrings(SimpleJob.MASTER_LABELS, masterLabels);
    String[] join = { "DDD", "EEE" };
    conf.setStrings(SimpleJob.JOIN_DATA_COLUMN, join);
    ValueCreator valueCreator = new JoinSomeRegexValueCreator(labels, false,
            "^(.*)\\t(.*)\\t(.*)\\t(.*)\\t(.*)$", true, simpleJoinMap, conf);
    Value value = new Value();

    String string1 = "a\tb\tc\tdaaaaD\teaaaaE";
    valueCreator.create(string1, value);
    assertEquals(value.getPrimitiveValue("AAA"), "a");
    assertEquals(value.getPrimitiveValue("BBB"), "b");
    assertEquals(value.getPrimitiveValue("CCC"), "c");
    assertEquals(value.getPrimitiveValue("DDD"), "daaaaD");
    assertEquals(value.getPrimitiveValue("EEE"), "eaaaaE");
    assertEquals(value.getPrimitiveValue("NAME"), "DdddD");

    value.clear();//from   www  . j  a  va  2  s  . c  o  m
    String string2 = "1\t2\t3\t41111IV\t51111V";
    valueCreator.create(string2, value);
    assertEquals(value.getPrimitiveValue("AAA"), "1");
    assertEquals(value.getPrimitiveValue("BBB"), "2");
    assertEquals(value.getPrimitiveValue("CCC"), "3");
    assertEquals(value.getPrimitiveValue("DDD"), "41111IV");
    assertEquals(value.getPrimitiveValue("EEE"), "51111V");
    assertEquals(value.getPrimitiveValue("NAME"), "IV");
}

---

From source file:org.huahinframework.core.lib.input.creator.JoinSomeValueCreatorTest.java

License:Apache License

@Test
public void testString() throws DataFormatException {
    String[] labels = { "AAA", "BBB", "CCC", "DDD", "EEE" };
    String[] masterLabels = { "DDD", "EEE", "NAME" };

    Map<List<String>, String[]> simpleJoinMap = new HashMap<List<String>, String[]>();
    String[] mv1 = { "d", "e", "DdddD" };
    simpleJoinMap.put(Arrays.asList("d", "e"), mv1);

    String[] mv2 = { "4", "5", "IV" };
    simpleJoinMap.put(Arrays.asList("4", "5"), mv2);

    Configuration conf = new Configuration();
    conf.setStrings(SimpleJob.MASTER_LABELS, masterLabels);
    String[] join = { "DDD", "EEE" };
    conf.setStrings(SimpleJob.JOIN_DATA_COLUMN, join);
    ValueCreator valueCreator = new JoinSomeValueCreator(labels, true, "\t", false, simpleJoinMap, conf);
    Value value = new Value();

    String string1 = "a\tb\tc\td\te";
    valueCreator.create(string1, value);
    assertEquals(value.getPrimitiveValue("AAA"), "a");
    assertEquals(value.getPrimitiveValue("BBB"), "b");
    assertEquals(value.getPrimitiveValue("CCC"), "c");
    assertEquals(value.getPrimitiveValue("DDD"), "d");
    assertEquals(value.getPrimitiveValue("EEE"), "e");
    assertEquals(value.getPrimitiveValue("NAME"), "DdddD");

    value.clear();/*from www. ja v  a2 s.co  m*/
    String string2 = "1\t2\t3\t4\t5";
    valueCreator.create(string2, value);
    assertEquals(value.getPrimitiveValue("AAA"), "1");
    assertEquals(value.getPrimitiveValue("BBB"), "2");
    assertEquals(value.getPrimitiveValue("CCC"), "3");
    assertEquals(value.getPrimitiveValue("DDD"), "4");
    assertEquals(value.getPrimitiveValue("EEE"), "5");
    assertEquals(value.getPrimitiveValue("NAME"), "IV");
}

---

From source file:org.huahinframework.core.lib.input.creator.JoinSomeValueCreatorTest.java

License:Apache License

@Test
public void testRegex() throws DataFormatException {
    String[] labels = { "AAA", "BBB", "CCC", "DDD", "EEE" };
    String[] masterLabels = { "DDD", "EEE", "NAME" };

    Map<List<String>, String[]> simpleJoinMap = new HashMap<List<String>, String[]>();
    String[] mv1 = { "d", "e", "DdddD" };
    simpleJoinMap.put(Arrays.asList("d", "e"), mv1);

    String[] mv2 = { "4", "5", "IV" };
    simpleJoinMap.put(Arrays.asList("4", "5"), mv2);

    Configuration conf = new Configuration();
    conf.setStrings(SimpleJob.MASTER_LABELS, masterLabels);
    String[] join = { "DDD", "EEE" };
    conf.setStrings(SimpleJob.JOIN_DATA_COLUMN, join);
    ValueCreator valueCreator = new JoinSomeValueCreator(labels, true, "^(.*)\\t(.*)\\t(.*)\\t(.*)\\t(.*)$",
            true, simpleJoinMap, conf);//from   ww w .  jav a  2 s .c o  m
    Value value = new Value();

    String string1 = "a\tb\tc\td\te";
    valueCreator.create(string1, value);
    assertEquals(value.getPrimitiveValue("AAA"), "a");
    assertEquals(value.getPrimitiveValue("BBB"), "b");
    assertEquals(value.getPrimitiveValue("CCC"), "c");
    assertEquals(value.getPrimitiveValue("DDD"), "d");
    assertEquals(value.getPrimitiveValue("EEE"), "e");
    assertEquals(value.getPrimitiveValue("NAME"), "DdddD");

    value.clear();
    String string2 = "1\t2\t3\t4\t5";
    valueCreator.create(string2, value);
    assertEquals(value.getPrimitiveValue("AAA"), "1");
    assertEquals(value.getPrimitiveValue("BBB"), "2");
    assertEquals(value.getPrimitiveValue("CCC"), "3");
    assertEquals(value.getPrimitiveValue("DDD"), "4");
    assertEquals(value.getPrimitiveValue("EEE"), "5");
    assertEquals(value.getPrimitiveValue("NAME"), "IV");
}

---

From source file:org.huahinframework.core.lib.input.creator.JoinValueCreatorTest.java

License:Apache License

@Test
public void testString() throws DataFormatException {
    String[] labels = { "AAA", "BBB", "CCC", "DDD" };
    String[] masterLabels = { "DDD", "NAME" };

    Map<String, String[]> simpleJoinMap = new HashMap<String, String[]>();
    String[] m1 = { "d", "DdddD" };
    simpleJoinMap.put("d", m1);
    String[] m2 = { "4", "IV" };
    simpleJoinMap.put("4", m2);

    Configuration conf = new Configuration();
    conf.setStrings(SimpleJob.MASTER_LABELS, masterLabels);
    conf.set(SimpleJob.JOIN_DATA_COLUMN, "DDD");
    ValueCreator valueCreator = new JoinValueCreator(labels, true, "\t", false, simpleJoinMap, conf);
    Value value = new Value();

    String string1 = "a\tb\tc\td";
    valueCreator.create(string1, value);
    assertEquals(value.getPrimitiveValue("AAA"), "a");
    assertEquals(value.getPrimitiveValue("BBB"), "b");
    assertEquals(value.getPrimitiveValue("CCC"), "c");
    assertEquals(value.getPrimitiveValue("DDD"), "d");
    assertEquals(value.getPrimitiveValue("NAME"), "DdddD");

    value.clear();//ww w  . j  a  v a 2s.c o m
    String string2 = "1\t2\t3\t4";
    valueCreator.create(string2, value);
    assertEquals(value.getPrimitiveValue("AAA"), "1");
    assertEquals(value.getPrimitiveValue("BBB"), "2");
    assertEquals(value.getPrimitiveValue("CCC"), "3");
    assertEquals(value.getPrimitiveValue("DDD"), "4");
    assertEquals(value.getPrimitiveValue("NAME"), "IV");
}

---

From source file:org.huahinframework.core.lib.input.creator.JoinValueCreatorTest.java

License:Apache License

@Test
public void testRegex() throws DataFormatException {
    String[] labels = { "AAA", "BBB", "CCC", "DDD" };
    String[] masterLabels = { "DDD", "NAME" };

    Map<String, String[]> simpleJoinMap = new HashMap<String, String[]>();
    String[] m1 = { "d", "DdddD" };
    simpleJoinMap.put("d", m1);
    String[] m2 = { "4", "IV" };
    simpleJoinMap.put("4", m2);

    Configuration conf = new Configuration();
    conf.setStrings(SimpleJob.MASTER_LABELS, masterLabels);
    conf.set(SimpleJob.JOIN_DATA_COLUMN, "DDD");
    ValueCreator valueCreator = new JoinValueCreator(labels, true, "^(.*)\\t(.*)\\t(.*)\\t(.*)$", true,
            simpleJoinMap, conf);//from   w  w w . j  av  a2 s.  c  o  m
    Value value = new Value();

    String string1 = "a\tb\tc\td";
    valueCreator.create(string1, value);
    assertEquals(value.getPrimitiveValue("AAA"), "a");
    assertEquals(value.getPrimitiveValue("BBB"), "b");
    assertEquals(value.getPrimitiveValue("CCC"), "c");
    assertEquals(value.getPrimitiveValue("DDD"), "d");
    assertEquals(value.getPrimitiveValue("NAME"), "DdddD");

    value.clear();
    String string2 = "1\t2\t3\t4";
    valueCreator.create(string2, value);
    assertEquals(value.getPrimitiveValue("AAA"), "1");
    assertEquals(value.getPrimitiveValue("BBB"), "2");
    assertEquals(value.getPrimitiveValue("CCC"), "3");
    assertEquals(value.getPrimitiveValue("DDD"), "4");
    assertEquals(value.getPrimitiveValue("NAME"), "IV");
}

---

From source file:org.jc.mrsqoophelper.main.SqoopHelperMain.java

public static void main(String[] args) {
    if (args == null || args.length < 6) {
        System.out.println("Incorrect usage of tool:");
        System.out.println(//from ww w  .  ja v  a 2s.c o  m
                "hadoop jar program.jar <package_name> <json_schema_file> <jar_classpath> <absolute_src_path> <field constraint output_file cast_to;field constraint output_path> <input_file_1;input_file_2;input_file_3> <output_file1;output_file2;...");
        return;
    }

    String packageName = args[0];
    String jsonPath = args[1];
    String classPath = args[2];
    String absolutePathOfSrc = args[3];
    //Pair field constraint ouput_file, do not join with logical operators.
    //Example: date gt 'yyyy/MM/dd' ouput_file
    //         date lt 'yyyy/MM/dd' output_file
    //This is correct!
    //But:
    //date gt 'yyyy/MM/dd' and date lt 'yyyy/MM/dd'
    //This is wrong!!
    //Conditions will be appended with AND
    //separate conditions with semi-colons.
    //conditions with same output file will be appended with AND operator
    String fieldToFilter = args[4];
    //separate input files by comma
    String inputAvroFiles = args[5];
    String outputAvroFiles = args[6];

    try {
        Path pathOfSchema = Paths.get(jsonPath);
        fieldToFilter = fieldToFilter.replace(CMD_LINE_TRIPLET_ORIGINAL_DELIM,
                TRIPLET_ELEMENTS_DELIMITER_SYMBOL);
        String schemaAsJson = Files.readAllLines(pathOfSchema, Charset.forName("UTF-8")).get(0);
        Class recordClass = Utils.ClassBuilder(schemaAsJson, packageName, classPath, absolutePathOfSrc);
        Configuration conf = new Configuration();

        conf.setStrings(FIELD_COND_OUTPUT_TRIPLET, fieldToFilter.split(";"));
        conf.set(TRIPLET_ELEMENTS_DELIMITER, TRIPLET_ELEMENTS_DELIMITER_SYMBOL);
        conf.set(FQCN_RECORD_CLASS, recordClass.getName());
        conf.set(AVRO_SCHEMA_AS_JSON, schemaAsJson);
        conf.set(SRC_ABSOLUTE_PATH, absolutePathOfSrc);
        conf.set(CLASS_ABSOLUTE_PATH, classPath);
        conf.set(PACKAGE_NAME, packageName);

        Job job = new Job(conf);
        //Set context for mapper and reducer.
        job.setJobName("Filter Records for Sqoop Export");
        job.setInputFormatClass(AvroKeyInputFormat.class);
        job.setMapperClass(Mapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        job.setReducerClass(Reducer.class);
        String[] argPaths = inputAvroFiles.split(";");
        org.apache.hadoop.fs.Path[] paths = new org.apache.hadoop.fs.Path[argPaths.length];
        for (int i = 0; i < argPaths.length; ++i) {
            paths[i] = new org.apache.hadoop.fs.Path(argPaths[i]);
        }

        org.apache.hadoop.mapreduce.lib.input.FileInputFormat.setInputPaths(job, paths);
        LazyOutputFormat.setOutputFormatClass(job, NullOutputFormat.class);

        //creating output files
        Schema avsc = new Schema.Parser().parse(schemaAsJson);
        DataFileWriter<Object> dfw = new DataFileWriter<>(new GenericDatumWriter<>(avsc));
        for (String out : outputAvroFiles.split(";")) {
            dfw.create(avsc, new File(out));
            dfw.close();
        }
        /*YarnLocalCluster yarnLocalCluster = new YarnLocalCluster.Builder()
        .setNumNodeManagers(1)
        .setNumLocalDirs(Integer.parseInt("1"))
        .setNumLogDirs(Integer.parseInt("1"))
        .setResourceManagerAddress("localhost")
        .setResourceManagerHostname("localhost:37001")
        .setResourceManagerSchedulerAddress("localhost:37002")
        .setResourceManagerResourceTrackerAddress("localhost:37003")
        .setResourceManagerWebappAddress("localhost:37004")
        .setUseInJvmContainerExecutor(true)
        .setConfig(conf)
        .build();
                
                
                
        yarnLocalCluster.start();*/

        System.exit(job.waitForCompletion(true) ? 0 : 1);
    } catch (Exception ex) {
        Logger.getLogger(SqoopHelperMain.class.getName()).log(Level.SEVERE, null, ex);
        System.out.println("Could not generate class for avro schema.");
    }

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• »