Word Count Example with Hadoop – 0.20

For detailed understanding on the working and control flow of this example refer

word count hadoop map reduce example

Mapper Class  - WordCountMapper.java

import java.io.IOException;

import java.util.StringTokenizer;

import org.apache.hadoop.io.*;

import org.apache.hadoop.mapreduce.Mapper;

public class WordCountMapper extends Mapper<LongWritable, Text, Text, IntWritable>

{

            //hadoop supported data types

      private final static IntWritable one = new IntWritable(1);

      private Text word = new Text();

         

           public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException

           {

             //taking one line at a time and tokenizing the same

               String line = value.toString();

               StringTokenizer tokenizer = newStringTokenizer(line);

           

             //iterating through all the words available in that line and forming the key value pair

               while (tokenizer.hasMoreTokens())

               {

                  word.set(tokenizer.nextToken());

                  //sending to output collector which inturn passes the same to reducer

                  context.write(word, one);

               }

           }

         

 }

Reducer Class - WordCountReducer.java

import java.io.IOException;

import java.util.Iterator;

import org.apache.hadoop.io.*;

import org.apache.hadoop.mapreduce.Reducer;

public class WordCountReducer extends Reducer<Text, IntWritable, Text, IntWritable>

{

      //Reduce method for just outputting the key from mapper as the value from mapper is just an empty string   

      public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException

      {

            int sum = 0;

            /*iterates through all the values available with a key and add them together and give the

            final result as the key and sum of its values*/

            for (IntWritable value : values)

            {

                  sum += value.get();

            }

            context.write(key, new IntWritable(sum));

       }

}

Driver Class - WordCount.java

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.conf.Configured;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.util.Tool;

import org.apache.hadoop.util.ToolRunner;

public class WordCount extends Configured implements Tool

{

      public int run(String[] args) throws Exception

      {

            //getting configuration object and setting job name

            Configuration conf = getConf();

        Job job = new Job(conf, "Word Count hadoop-0.20");

      

        //setting the class names

        job.setJarByClass(WordCount.class);

        job.setMapperClass(WordCountMapper.class);

        job.setReducerClass(WordCountReducer.class);

        //setting the output data type classes

        job.setOutputKeyClass(Text.class);

        job.setOutputValueClass(IntWritable.class);

        //to accept the hdfs input and outpur dir at run time

        FileInputFormat.addInputPath(job, new Path(args[0]));

        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        return job.waitForCompletion(true) ? 0 : 1;

    }

    public static void main(String[] args) throws Exception {

        int res = ToolRunner.run(new Configuration(), newWordCount(), args);

        System.exit(res);

    }

}