The mapper
Sunday, February 28, 2010
8:43 PM
The mapper
Input: shards (fragments) of the text to be counted.
Output: a list of <w,1> pairs for w a word.
// a value of 1 of the appropriate type for reducing
private final IntWritable one = new IntWritable(1);
public void map(WritableComparable key, Writable value,
  OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
  // WritableComparable means one can apply >, =, < to it. 
  // Writable means that one can write it, 
  // but might *not* be able to compare it. 
  // WritableComparable is a subinterface of Writable. 
  // each line is a shard, as a string. 
  String line = value.toString();
  // break line into a sequence of tokens
  StringTokenizer itr = new StringTokenizer(line.toLowerCase());
  // iterate over tokens. 
  while(itr.hasMoreTokens()) {
   word.set(itr.nextToken());
   output.collect(word, one); // add one!
  }
 }