Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. */ package org.apache.avro.mapreduce; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.HashMap; import java.util.Map; import org.apache.avro.Schema; import org.apache.avro.file.DataFileReader; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.mapred.AvroKey; import org.apache.avro.mapred.AvroValue; import org.apache.avro.mapred.FsInput; import org.apache.avro.specific.SpecificDatumReader; import org.apache.avro.reflect.ReflectData; import org.apache.avro.reflect.ReflectDatumReader; import org.apache.avro.util.Utf8; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; public class TestWordCount { @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); public static final Schema STATS_SCHEMA = Schema.parse("{\"name\":\"stats\",\"type\":\"record\"," + "\"fields\":[{\"name\":\"count\",\"type\":\"int\"}," + "{\"name\":\"name\",\"type\":\"string\"}]}"); public static class ReflectStats { String name; int count; } // permit data written as SpecficStats to be read as ReflectStats private static Schema REFLECT_STATS_SCHEMA = ReflectData.get().getSchema(ReflectStats.class); static { REFLECT_STATS_SCHEMA.addAlias(TextStats.SCHEMA$.getFullName()); } private static class LineCountMapper extends Mapper<LongWritable, Text, Text, IntWritable> { private IntWritable mOne; @Override protected void setup(Context context) { mOne = new IntWritable(1); } @Override protected void map(LongWritable fileByteOffset, Text line, Context context) throws IOException, InterruptedException { context.write(line, mOne); } } private static class StatCountMapper extends Mapper<AvroKey<TextStats>, NullWritable, Text, IntWritable> { private IntWritable mCount; private Text mText; @Override protected void setup(Context context) { mCount = new IntWritable(0); mText = new Text(""); } @Override protected void map(AvroKey<TextStats> record, NullWritable ignore, Context context) throws IOException, InterruptedException { mCount.set(record.datum().count); mText.set(record.datum().name.toString()); context.write(mText, mCount); } } private static class ReflectCountMapper extends Mapper<AvroKey<ReflectStats>, NullWritable, Text, IntWritable> { private IntWritable mCount; private Text mText; @Override protected void setup(Context context) { mCount = new IntWritable(0); mText = new Text(""); } @Override protected void map(AvroKey<ReflectStats> record, NullWritable ignore, Context context) throws IOException, InterruptedException { mCount.set(record.datum().count); mText.set(record.datum().name); context.write(mText, mCount); } } private static class AvroSumReducer extends Reducer<Text, IntWritable, AvroKey<CharSequence>, AvroValue<Integer>> { @Override protected void reduce(Text key, Iterable<IntWritable> counts, Context context) throws IOException, InterruptedException { int sum = 0; for (IntWritable count : counts) { sum += count.get(); } context.write(new AvroKey<CharSequence>(key.toString()), new AvroValue<Integer>(sum)); } } private static class GenericStatsReducer extends Reducer<Text, IntWritable, AvroKey<GenericData.Record>, NullWritable> { private AvroKey<GenericData.Record> mStats; @Override protected void setup(Context context) { mStats = new AvroKey<GenericData.Record>(null); } @Override protected void reduce(Text line, Iterable<IntWritable> counts, Context context) throws IOException, InterruptedException { GenericData.Record record = new GenericData.Record(STATS_SCHEMA); int sum = 0; for (IntWritable count : counts) { sum += count.get(); } record.put("name", new Utf8(line.toString())); record.put("count", new Integer(sum)); mStats.datum(record); context.write(mStats, NullWritable.get()); } } private static class SpecificStatsReducer extends Reducer<Text, IntWritable, AvroKey<TextStats>, NullWritable> { private AvroKey<TextStats> mStats; @Override protected void setup(Context context) { mStats = new AvroKey<TextStats>(null); } @Override protected void reduce(Text line, Iterable<IntWritable> counts, Context context) throws IOException, InterruptedException { TextStats record = new TextStats(); record.count = 0; for (IntWritable count : counts) { record.count += count.get(); } record.name = line.toString(); mStats.datum(record); context.write(mStats, NullWritable.get()); } } private static class ReflectStatsReducer extends Reducer<Text, IntWritable, AvroKey<ReflectStats>, NullWritable> { private AvroKey<ReflectStats> mStats; @Override protected void setup(Context context) { mStats = new AvroKey<ReflectStats>(null); } @Override protected void reduce(Text line, Iterable<IntWritable> counts, Context context) throws IOException, InterruptedException { ReflectStats record = new ReflectStats(); record.count = 0; for (IntWritable count : counts) { record.count += count.get(); } record.name = line.toString(); mStats.datum(record); context.write(mStats, NullWritable.get()); } } private static class SortMapper extends Mapper<AvroKey<TextStats>, NullWritable, AvroKey<TextStats>, NullWritable> { @Override protected void map(AvroKey<TextStats> key, NullWritable value, Context context) throws IOException, InterruptedException { context.write(key, value); } } private static class SortReducer extends Reducer<AvroKey<TextStats>, NullWritable, AvroKey<TextStats>, NullWritable> { @Override protected void reduce(AvroKey<TextStats> key, Iterable<NullWritable> ignore, Context context) throws IOException, InterruptedException { context.write(key, NullWritable.get()); } } @Test public void testAvroGenericOutput() throws Exception { Job job = new Job(); FileInputFormat.setInputPaths(job, new Path( getClass().getResource("/org/apache/avro/mapreduce/mapreduce-test-input.txt").toURI().toString())); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(LineCountMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setReducerClass(GenericStatsReducer.class); AvroJob.setOutputKeySchema(job, STATS_SCHEMA); job.setOutputFormatClass(AvroKeyOutputFormat.class); Path outputPath = new Path(tmpFolder.getRoot().getPath() + "/out-generic"); FileOutputFormat.setOutputPath(job, outputPath); Assert.assertTrue(job.waitForCompletion(true)); // Check that the results from the MapReduce were as expected. FileSystem fileSystem = FileSystem.get(job.getConfiguration()); FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/part-*")); Assert.assertEquals(1, outputFiles.length); DataFileReader<GenericData.Record> reader = new DataFileReader<GenericData.Record>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new GenericDatumReader<GenericData.Record>(STATS_SCHEMA)); Map<String, Integer> counts = new HashMap<String, Integer>(); for (GenericData.Record record : reader) { counts.put(((Utf8) record.get("name")).toString(), (Integer) record.get("count")); } reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); } @Test public void testAvroSpecificOutput() throws Exception { Job job = new Job(); FileInputFormat.setInputPaths(job, new Path( getClass().getResource("/org/apache/avro/mapreduce/mapreduce-test-input.txt").toURI().toString())); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(LineCountMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setReducerClass(SpecificStatsReducer.class); AvroJob.setOutputKeySchema(job, TextStats.SCHEMA$); job.setOutputFormatClass(AvroKeyOutputFormat.class); Path outputPath = new Path(tmpFolder.getRoot().getPath() + "/out-specific"); FileOutputFormat.setOutputPath(job, outputPath); Assert.assertTrue(job.waitForCompletion(true)); // Check that the results from the MapReduce were as expected. FileSystem fileSystem = FileSystem.get(job.getConfiguration()); FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/part-*")); Assert.assertEquals(1, outputFiles.length); DataFileReader<TextStats> reader = new DataFileReader<TextStats>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new SpecificDatumReader<TextStats>()); Map<String, Integer> counts = new HashMap<String, Integer>(); for (TextStats record : reader) { counts.put(record.name.toString(), record.count); } reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); } @Test public void testAvroReflectOutput() throws Exception { Job job = new Job(); FileInputFormat.setInputPaths(job, new Path( getClass().getResource("/org/apache/avro/mapreduce/mapreduce-test-input.txt").toURI().toString())); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(LineCountMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setReducerClass(ReflectStatsReducer.class); AvroJob.setOutputKeySchema(job, REFLECT_STATS_SCHEMA); job.setOutputFormatClass(AvroKeyOutputFormat.class); Path outputPath = new Path(tmpFolder.getRoot().getPath() + "/out-reflect"); FileOutputFormat.setOutputPath(job, outputPath); Assert.assertTrue(job.waitForCompletion(true)); // Check that the results from the MapReduce were as expected. FileSystem fileSystem = FileSystem.get(job.getConfiguration()); FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/part-*")); Assert.assertEquals(1, outputFiles.length); DataFileReader<ReflectStats> reader = new DataFileReader<ReflectStats>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new ReflectDatumReader<ReflectStats>()); Map<String, Integer> counts = new HashMap<String, Integer>(); for (ReflectStats record : reader) { counts.put(record.name.toString(), record.count); } reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); } @Test public void testAvroInput() throws Exception { Job job = new Job(); FileInputFormat.setInputPaths(job, new Path( getClass().getResource("/org/apache/avro/mapreduce/mapreduce-test-input.avro").toURI().toString())); job.setInputFormatClass(AvroKeyInputFormat.class); AvroJob.setInputKeySchema(job, TextStats.SCHEMA$); job.setMapperClass(StatCountMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setReducerClass(SpecificStatsReducer.class); AvroJob.setOutputKeySchema(job, TextStats.SCHEMA$); job.setOutputFormatClass(AvroKeyOutputFormat.class); Path outputPath = new Path(tmpFolder.getRoot().getPath() + "/out-specific-input"); FileOutputFormat.setOutputPath(job, outputPath); Assert.assertTrue(job.waitForCompletion(true)); // Check that the results from the MapReduce were as expected. FileSystem fileSystem = FileSystem.get(job.getConfiguration()); FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/part-*")); Assert.assertEquals(1, outputFiles.length); DataFileReader<TextStats> reader = new DataFileReader<TextStats>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new SpecificDatumReader<TextStats>()); Map<String, Integer> counts = new HashMap<String, Integer>(); for (TextStats record : reader) { counts.put(record.name.toString(), record.count); } reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); } @Test public void testReflectInput() throws Exception { Job job = new Job(); FileInputFormat.setInputPaths(job, new Path( getClass().getResource("/org/apache/avro/mapreduce/mapreduce-test-input.avro").toURI().toString())); job.setInputFormatClass(AvroKeyInputFormat.class); AvroJob.setInputKeySchema(job, REFLECT_STATS_SCHEMA); job.setMapperClass(ReflectCountMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setReducerClass(ReflectStatsReducer.class); AvroJob.setOutputKeySchema(job, REFLECT_STATS_SCHEMA); job.setOutputFormatClass(AvroKeyOutputFormat.class); Path outputPath = new Path(tmpFolder.getRoot().getPath() + "/out-reflect-input"); FileOutputFormat.setOutputPath(job, outputPath); Assert.assertTrue(job.waitForCompletion(true)); // Check that the results from the MapReduce were as expected. FileSystem fileSystem = FileSystem.get(job.getConfiguration()); FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/part-*")); Assert.assertEquals(1, outputFiles.length); DataFileReader<ReflectStats> reader = new DataFileReader<ReflectStats>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new ReflectDatumReader<ReflectStats>()); Map<String, Integer> counts = new HashMap<String, Integer>(); for (ReflectStats record : reader) { counts.put(record.name.toString(), record.count); } reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); } @Test public void testAvroMapOutput() throws Exception { Job job = new Job(); FileInputFormat.setInputPaths(job, new Path( getClass().getResource("/org/apache/avro/mapreduce/mapreduce-test-input.avro").toURI().toString())); job.setInputFormatClass(AvroKeyInputFormat.class); AvroJob.setInputKeySchema(job, TextStats.SCHEMA$); job.setMapperClass(SortMapper.class); AvroJob.setMapOutputKeySchema(job, TextStats.SCHEMA$); job.setMapOutputValueClass(NullWritable.class); job.setReducerClass(SortReducer.class); AvroJob.setOutputKeySchema(job, TextStats.SCHEMA$); job.setOutputFormatClass(AvroKeyOutputFormat.class); Path outputPath = new Path(tmpFolder.getRoot().getPath() + "/out-specific-input"); FileOutputFormat.setOutputPath(job, outputPath); Assert.assertTrue(job.waitForCompletion(true)); // Check that the results from the MapReduce were as expected. FileSystem fileSystem = FileSystem.get(job.getConfiguration()); FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/part-*")); Assert.assertEquals(1, outputFiles.length); DataFileReader<TextStats> reader = new DataFileReader<TextStats>( new FsInput(outputFiles[0].getPath(), job.getConfiguration()), new SpecificDatumReader<TextStats>()); Map<String, Integer> counts = new HashMap<String, Integer>(); for (TextStats record : reader) { counts.put(record.name.toString(), record.count); } reader.close(); Assert.assertEquals(3, counts.get("apple").intValue()); Assert.assertEquals(2, counts.get("banana").intValue()); Assert.assertEquals(1, counts.get("carrot").intValue()); } /** * Tests the MR output to text files when using AvroKey and AvroValue records. */ @Test public void testAvroUsingTextFileOutput() throws Exception { Job job = new Job(); FileInputFormat.setInputPaths(job, new Path( getClass().getResource("/org/apache/avro/mapreduce/mapreduce-test-input.txt").toURI().toString())); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(LineCountMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setReducerClass(AvroSumReducer.class); AvroJob.setOutputKeySchema(job, Schema.create(Schema.Type.STRING)); AvroJob.setOutputValueSchema(job, Schema.create(Schema.Type.INT)); job.setOutputFormatClass(TextOutputFormat.class); Path outputPath = new Path(tmpFolder.getRoot().getPath() + "/out-text"); FileOutputFormat.setOutputPath(job, outputPath); Assert.assertTrue(job.waitForCompletion(true)); // Check that the results from the MapReduce were as expected. FileSystem fileSystem = FileSystem.get(job.getConfiguration()); FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/part-*")); Assert.assertEquals(1, outputFiles.length); Path filePath = outputFiles[0].getPath(); InputStream inputStream = filePath.getFileSystem(job.getConfiguration()).open(filePath); Assert.assertNotNull(inputStream); BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); try { Assert.assertTrue(reader.ready()); Assert.assertEquals("apple\t3", reader.readLine()); Assert.assertEquals("banana\t2", reader.readLine()); Assert.assertEquals("carrot\t1", reader.readLine()); Assert.assertFalse(reader.ready()); } finally { reader.close(); } } }