Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.avro.mapreduce; import java.io.IOException; import java.util.HashMap; import java.util.Map; import org.apache.avro.Schema; import org.apache.avro.file.DataFileReader; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.reflect.ReflectDatumReader; import org.apache.avro.mapred.AvroKey; import org.apache.avro.mapred.FsInput; import org.apache.avro.specific.SpecificDatumReader; import org.apache.avro.util.Utf8; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.apache.avro.mapred.Pair; public class TestAvroMultipleOutputs { @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); public static final Schema STATS_SCHEMA = Schema.parse("{\"name\":\"stats\",\"type\":\"record\"," + "\"fields\":[{\"name\":\"count\",\"type\":\"int\"}," + "{\"name\":\"name\",\"type\":\"string\"}]}"); public static final Schema STATS_SCHEMA_2 = Schema.parse("{\"name\":\"stats\",\"type\":\"record\"," + "\"fields\":[{\"name\":\"count1\",\"type\":\"int\"}," + "{\"name\":\"name1\",\"type\":\"string\"}]}"); private static class LineCountMapper extends Mapper<LongWritable, Text, Text, IntWritable> { private IntWritable mOne; @Override protected void setup(Context context) { mOne = new IntWritable(1); } @Override protected void map(LongWritable fileByteOffset, Text line, Context context) throws IOException, InterruptedException { context.write(line, mOne); } } private static class StatCountMapper extends Mapper<AvroKey<TextStats>, NullWritable, Text, IntWritable> { private IntWritable mCount; private Text mText; @Override protected void setup(Context context) { mCount = new IntWritable(0); mText = new Text(""); } @Override protected void map(AvroKey<TextStats> record, NullWritable ignore, Context context) throws IOException, InterruptedException { mCount.set(record.datum().count); mText.set(record.datum().name.toString()); context.write(mText, mCount); } } private static class GenericStatsReducer extends Reducer<Text, IntWritable, AvroKey<GenericData.Record>, NullWritable> { private AvroKey<GenericData.Record> mStats; private AvroMultipleOutputs amos; @Override protected void setup(Context context) { mStats = new AvroKey<GenericData.Record>(null); amos = new AvroMultipleOutputs(context); } @Override protected void reduce(Text line, Iterable<IntWritable> counts, Context context) throws IOException, InterruptedException { GenericData.Record record = new GenericData.Record(STATS_SCHEMA); GenericData.Record record2 = new GenericData.Record(STATS_SCHEMA_2); int sum = 0; for (IntWritable count : counts) { sum += count.get(); } record.put("name", new Utf8(line.toString())); record.put("count", new Integer(sum)); mStats.datum(record); context.write(mStats, NullWritable.get()); amos.write("myavro", mStats, NullWritable.get()); record2.put("name1", new Utf8(line.toString())); record2.put("count1", new Integer(sum)); mStats.datum(record2); amos.write(mStats, NullWritable.get(), STATS_SCHEMA_2, null, "testnewwrite2"); amos.write("myavro1", mStats); amos.write(mStats, NullWritable.get(), STATS_SCHEMA, null, "testnewwrite"); amos.write(mStats, NullWritable.get(), "testwritenonschema"); } @Override protected void cleanup(Context context) throws IOException, InterruptedException { amos.close(); } } private static class SpecificStatsReducer extends Reducer<Text, IntWritable, AvroKey<TextStats>, NullWritable> { private AvroKey<TextStats> mStats; private AvroMultipleOutputs amos; @Override protected void setup(Context context) { mStats = new AvroKey<TextStats>(null); amos = new AvroMultipleOutputs(context); } @Override protected void reduce(Text line, Iterable<IntWritable> counts, Context context) throws IOException, InterruptedException { TextStats record = new TextStats(); record.count = 0; for (IntWritable count : counts) { record.count += count.get(); } record.name = line.toString(); mStats.datum(record); context.write(mStats, NullWritable.get()); amos.write("myavro3", mStats, NullWritable.get()); } @Override protected void cleanup(Context context) throws IOException, InterruptedException { amos.close(); } } private static class SortMapper extends Mapper<AvroKey<TextStats>, NullWritable, AvroKey<TextStats>, NullWritable> { @Override protected void map(AvroKey<TextStats> key, NullWritable value, Context context) throws IOException, InterruptedException { context.write(key, value); } } private static class SortReducer extends Reducer<AvroKey<TextStats>, NullWritable, AvroKey<TextStats>, NullWritable> { @Override protected void reduce(AvroKey<TextStats> key, Iterable<NullWritable> ignore, Context context) throws IOException, InterruptedException { context.write(key, NullWritable.get()); } } @Test public void testAvroGenericOutput() throws Exception { Job job = new Job(); FileInputFormat.setInputPaths(job, new Path( getClass().getResource("/org/apache/avro/mapreduce/mapreduce-test-input.txt").toURI().toString())); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(LineCountMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setReducerClass(GenericStatsReducer.class); AvroJob.setOutputKeySchema(job, STATS_SCHEMA); AvroMultipleOutputs.addNamedOutput(job, "myavro", AvroKeyOutputFormat.class, STATS_SCHEMA, null); AvroMultipleOutputs.addNamedOutput(job, "myavro1", AvroKeyOutputFormat.class, STATS_SCHEMA_2); job.setOutputFormatClass(AvroKeyOutputFormat.class); String dir = System.getProperty("test.dir", ".") + "/mapred"; Path outputPath = new Path(dir + "/out"); outputPath.getFileSystem(job.getConfiguration()).delete(outputPath); FileOutputFormat.setOutputPath(job, outputPath); Assert.assertTrue(job.waitForCompletion(true)); // Check that the results from the MapReduce were as expected. FileSystem fileSystem = FileSystem.get(job.getConfiguration()); FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/myavro-r-00000.avro")); Assert.assertEquals(1, outputFiles.length); DataFileReader<GenericData.Record> reader = new DataFileReader<GenericData.Record>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new GenericDatumReader<GenericData.Record>(STATS_SCHEMA)); Map<String, Integer> counts = new HashMap<String, Integer>(); for (GenericData.Record record : reader) { counts.put(((Utf8) record.get("name")).toString(), (Integer) record.get("count")); } reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); outputFiles = fileSystem.globStatus(outputPath.suffix("/myavro1-r-00000.avro")); Assert.assertEquals(1, outputFiles.length); reader = new DataFileReader<GenericData.Record>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new GenericDatumReader<GenericData.Record>(STATS_SCHEMA_2)); counts = new HashMap<String, Integer>(); for (GenericData.Record record : reader) { counts.put(((Utf8) record.get("name1")).toString(), (Integer) record.get("count1")); } reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); outputFiles = fileSystem.globStatus(outputPath.suffix("/testnewwrite-r-00000.avro")); Assert.assertEquals(1, outputFiles.length); reader = new DataFileReader<GenericData.Record>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new GenericDatumReader<GenericData.Record>(STATS_SCHEMA)); counts = new HashMap<String, Integer>(); for (GenericData.Record record : reader) { counts.put(((Utf8) record.get("name")).toString(), (Integer) record.get("count")); } reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); outputFiles = fileSystem.globStatus(outputPath.suffix("/testnewwrite2-r-00000.avro")); Assert.assertEquals(1, outputFiles.length); reader = new DataFileReader<GenericData.Record>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new GenericDatumReader<GenericData.Record>(STATS_SCHEMA_2)); counts = new HashMap<String, Integer>(); for (GenericData.Record record : reader) { counts.put(((Utf8) record.get("name1")).toString(), (Integer) record.get("count1")); } reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); outputFiles = fileSystem.globStatus(outputPath.suffix("/testwritenonschema-r-00000.avro")); Assert.assertEquals(1, outputFiles.length); reader = new DataFileReader<GenericData.Record>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new GenericDatumReader<GenericData.Record>(STATS_SCHEMA)); counts = new HashMap<String, Integer>(); for (GenericData.Record record : reader) { counts.put(((Utf8) record.get("name")).toString(), (Integer) record.get("count")); } reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); } @Test public void testAvroSpecificOutput() throws Exception { Job job = new Job(); FileInputFormat.setInputPaths(job, new Path( getClass().getResource("/org/apache/avro/mapreduce/mapreduce-test-input.txt").toURI().toString())); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(LineCountMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); AvroMultipleOutputs.addNamedOutput(job, "myavro3", AvroKeyOutputFormat.class, TextStats.SCHEMA$, null); job.setReducerClass(SpecificStatsReducer.class); AvroJob.setOutputKeySchema(job, TextStats.SCHEMA$); job.setOutputFormatClass(AvroKeyOutputFormat.class); String dir = System.getProperty("test.dir", ".") + "/mapred"; Path outputPath = new Path(dir + "/out-specific"); outputPath.getFileSystem(job.getConfiguration()).delete(outputPath); FileOutputFormat.setOutputPath(job, outputPath); Assert.assertTrue(job.waitForCompletion(true)); FileSystem fileSystem = FileSystem.get(job.getConfiguration()); FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/myavro3-*")); Assert.assertEquals(1, outputFiles.length); DataFileReader<TextStats> reader = new DataFileReader<TextStats>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new SpecificDatumReader<TextStats>()); Map<String, Integer> counts = new HashMap<String, Integer>(); for (TextStats record : reader) { counts.put(record.name.toString(), record.count); } reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); } @Test public void testAvroInput() throws Exception { Job job = new Job(); FileInputFormat.setInputPaths(job, new Path( getClass().getResource("/org/apache/avro/mapreduce/mapreduce-test-input.avro").toURI().toString())); job.setInputFormatClass(AvroKeyInputFormat.class); AvroJob.setInputKeySchema(job, TextStats.SCHEMA$); AvroMultipleOutputs.addNamedOutput(job, "myavro3", AvroKeyOutputFormat.class, TextStats.SCHEMA$, null); job.setMapperClass(StatCountMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setReducerClass(SpecificStatsReducer.class); AvroJob.setOutputKeySchema(job, TextStats.SCHEMA$); job.setOutputFormatClass(AvroKeyOutputFormat.class); Path outputPath = new Path(tmpFolder.getRoot().getPath() + "/out-specific-input"); FileOutputFormat.setOutputPath(job, outputPath); Assert.assertTrue(job.waitForCompletion(true)); // Check that the results from the MapReduce were as expected. FileSystem fileSystem = FileSystem.get(job.getConfiguration()); FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/myavro3-*")); Assert.assertEquals(1, outputFiles.length); DataFileReader<TextStats> reader = new DataFileReader<TextStats>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new SpecificDatumReader<TextStats>()); Map<String, Integer> counts = new HashMap<String, Integer>(); for (TextStats record : reader) { counts.put(record.name.toString(), record.count); } reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); } @Test public void testAvroMapOutput() throws Exception { Job job = new Job(); FileInputFormat.setInputPaths(job, new Path( getClass().getResource("/org/apache/avro/mapreduce/mapreduce-test-input.avro").toURI().toString())); job.setInputFormatClass(AvroKeyInputFormat.class); AvroJob.setInputKeySchema(job, TextStats.SCHEMA$); job.setMapperClass(SortMapper.class); AvroJob.setMapOutputKeySchema(job, TextStats.SCHEMA$); job.setMapOutputValueClass(NullWritable.class); job.setReducerClass(SortReducer.class); AvroJob.setOutputKeySchema(job, TextStats.SCHEMA$); job.setOutputFormatClass(AvroKeyOutputFormat.class); Path outputPath = new Path(tmpFolder.getRoot().getPath() + "/out-specific-input"); FileOutputFormat.setOutputPath(job, outputPath); Assert.assertTrue(job.waitForCompletion(true)); // Check that the results from the MapReduce were as expected. FileSystem fileSystem = FileSystem.get(job.getConfiguration()); FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/part-*")); Assert.assertEquals(1, outputFiles.length); DataFileReader<TextStats> reader = new DataFileReader<TextStats>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new SpecificDatumReader<TextStats>()); Map<String, Integer> counts = new HashMap<String, Integer>(); for (TextStats record : reader) { counts.put(record.name.toString(), record.count); } reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); } }