Java tutorial
/* * Copyright 2015-2016 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package co.cask.cdap.examples.sportresults; import co.cask.cdap.api.Resources; import co.cask.cdap.api.data.batch.Input; import co.cask.cdap.api.data.batch.Output; import co.cask.cdap.api.dataset.lib.FileSetArguments; import co.cask.cdap.api.dataset.lib.PartitionFilter; import co.cask.cdap.api.dataset.lib.PartitionKey; import co.cask.cdap.api.dataset.lib.PartitionedFileSet; import co.cask.cdap.api.dataset.lib.PartitionedFileSetArguments; import co.cask.cdap.api.mapreduce.AbstractMapReduce; import co.cask.cdap.api.mapreduce.MapReduceContext; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.util.Map; /** * A MapReduce program that reads game results and counts statistics per team. */ public class ScoreCounter extends AbstractMapReduce { private static final Logger LOG = LoggerFactory.getLogger(ScoreCounter.class); @Override public void configure() { setDescription("reads game results and counts statistics per team"); setMapperResources(new Resources(512)); } @Override public void beforeSubmit(MapReduceContext context) throws Exception { Job job = context.getHadoopJob(); job.setMapperClass(ResultsMapper.class); job.setReducerClass(TeamCounter.class); job.setNumReduceTasks(1); String league = context.getRuntimeArguments().get("league"); Preconditions.checkNotNull(league); // Configure the input to read all seasons for the league Map<String, String> inputArgs = Maps.newHashMap(); PartitionedFileSetArguments.setInputPartitionFilter(inputArgs, PartitionFilter.builder().addValueCondition("league", league).build()); context.addInput(Input.ofDataset("results", inputArgs)); // Each run writes its output to a partition for the league Map<String, String> outputArgs = Maps.newHashMap(); PartitionKey outputKey = PartitionKey.builder().addStringField("league", league).build(); PartitionedFileSetArguments.setOutputPartitionKey(outputArgs, outputKey); context.addOutput(Output.ofDataset("totals", outputArgs)); // used only for logging: PartitionedFileSet input = context.getDataset("results", inputArgs); PartitionedFileSet outputFileSet = context.getDataset("totals", outputArgs); String outputPath = FileSetArguments .getOutputPath(outputFileSet.getEmbeddedFileSet().getRuntimeArguments()); LOG.info("input: {}, output: {}", input.getEmbeddedFileSet().getInputLocations(), outputPath); } /** * The Mapper emits a record with the team name, points scored, and points conceded, for both teams. */ public static class ResultsMapper extends Mapper<LongWritable, Text, Text, GameStat> { @Override protected void map(LongWritable position, Text value, Context context) throws IOException, InterruptedException { String[] fields = value.toString().split(","); if (fields.length < 5) { return; } String winner = fields[1]; String loser = fields[2]; try { int winnerPoints = Integer.parseInt(fields[3]); int loserPoints = Integer.parseInt(fields[4]); context.write(new Text(winner), new GameStat(winnerPoints, loserPoints)); context.write(new Text(loser), new GameStat(loserPoints, winnerPoints)); } catch (NumberFormatException e) { LOG.debug("Exception parsing input position {}: {}", position, value.toString()); } } } /** * The reducer counts all the different statistics. */ public static class TeamCounter extends Reducer<Text, GameStat, Text, String> { @Override protected void reduce(Text key, Iterable<GameStat> values, Context context) throws IOException, InterruptedException { int losses = 0, wins = 0, ties = 0, scored = 0, conceded = 0; for (GameStat stat : values) { if (stat.getScored() > stat.getConceded()) { wins++; } else if (stat.getScored() < stat.getConceded()) { losses++; } else { ties++; } scored += stat.getScored(); conceded += stat.getConceded(); } context.write(key, String.format("%d,%d,%d,%d,%d", wins, ties, losses, scored, conceded)); } } /** * Private writable helper class used between mappers and reducers. */ private static class GameStat implements Writable { private int scored; private int conceded; @SuppressWarnings("unused") GameStat() { } GameStat(int scored, int conceded) { this.scored = scored; this.conceded = conceded; } public int getScored() { return scored; } public int getConceded() { return conceded; } @Override public void write(DataOutput out) throws IOException { out.writeInt(scored); out.writeInt(conceded); } @Override public void readFields(DataInput in) throws IOException { scored = in.readInt(); conceded = in.readInt(); } } }