MapReduce编程实例之source-destination-count-avertime

来源：互联网发布：ios编程语言编辑：程序博客网时间：2024/06/28 10:53
这里写图片描述
//Hw2Part1.java
/** * Statistic the average time and the number of communication between source and destination  * Author:Rosun * date: 2017/4/8 */import java.io.IOException;import java.math.*;import java.util.StringTokenizer;import java.io.DataInput;import java.io.DataOutput;import org.apache.hadoop.io.Writable;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.util.GenericOptionsParser;/** * set value of count and communication's time * */class MymulvalueWritable implements Writable{    //value     public int count;    public float avertime;    public void set(int count,float avertime){        this.count=count;        this.avertime=avertime;    }    public int getCount(){        return this.count;    }    public float getAvertime(){        return this.avertime;    }    @Override    public String toString(){        return count+"\t"+avertime+"\t";    }    @Override    public void readFields(DataInput in )throws IOException{        this.count=in.readInt();        this.avertime=in.readFloat();    }    @Override    public void write(DataOutput out )throws IOException{        out.writeInt(count);        out.writeFloat(avertime);    }}public class Hw2Part1{    public Hw2Part1(){        System.out.println("excute constructed function--Hw2Part1()");    }  /**   * this class extends Mapper,split input date ,(ik,iv)->(mk,mv)   * @author guest   */  public static class TokenizerMapper                 extends Mapper<Object, Text, Text, MymulvalueWritable>{    public MymulvalueWritable result = new MymulvalueWritable();    private Text source_destination = new Text();    public void map(Object key, Text value, Context context                    ) throws IOException, InterruptedException {          System.out.println("Excute map() in TokenizerMapper Class");          //get every line          StringTokenizer itr = new StringTokenizer(value.toString(),"\n");          while (itr.hasMoreTokens()) {              String[] itr1=itr.nextToken().split("\\s+");//get strings[] from itr              //record format legal              try{                  if(itr1.length==3 && Float.parseFloat(itr1[2])>=0.0 &&itr1[2].matches("\\d+(.\\d+)?[fF]?")){                      System.out.println(itr1[0]+"     "+itr1[1]);                      source_destination.set(itr1[0]+"     "+itr1[1]);//m-key                        //System.out.println("time="+Float.parseFloat(itr1[2]));//holding time                       result.set(1,Float.parseFloat(itr1[2]));//m-value  count time                      context.write(source_destination,result);                      System.out.println(result.toString());                  }              }catch(Exception e){              }              else{                   // throw away the record               }          }    }  }  /**   * this class extens Reducer Class,statistic (mk,list(mv) ) on single block   * @author guest   *   */  public static class IntSumCombiner                extends Reducer<Text,MymulvalueWritable,Text,MymulvalueWritable> {        //  System.out.println("into IntSumCombiner");        public  MymulvalueWritable result =new MymulvalueWritable();        public void reduce(Text key, Iterable<MymulvalueWritable> values,Context context)                     throws IOException, InterruptedException {              float aver = 0.0f;        System.out.println("into IntSumCombiner");              int i=0;              System.out.println("Excute reduce() in IntSumCombiner Class");              System.out.println("Excute sum by key");              for (MymulvalueWritable val : values) {                  aver += val.getAvertime();                  i+=val.getCount();              }              //if(i!=0)                //  aver=aver/i;              System.out.println("key="+key.toString());              //count and avertime              result.set(i,aver);              context.write(key, result);              System.out.println(result.toString());        }  }  /**   * this class extends Reducer: (mk,mv)->(ok,ov)   * @author guest   */  public static class IntSumReducer                extends Reducer<Text,MymulvalueWritable,Text,Text> {        private Text result_key= new Text();        private Text result_value= new Text();        public void reduce(Text key, Iterable<MymulvalueWritable> values, Context context)                    throws IOException, InterruptedException {              float aver = 0.0f;              int i=0;              System.out.println("Excute reduce() in IntSumReducer Class");              //sum by key              for (MymulvalueWritable val : values) {                //aver+= val.getAvertime()*val.getCount();                  aver+= val.getAvertime();                i+=val.getCount();              }              //get the average holding time              if(i!=0)                  aver=aver/i;              //keep 3 decimal places              BigDecimal b=new BigDecimal(aver);              float aver1=b.setScale(3,BigDecimal.ROUND_HALF_UP).floatValue();//convert 0.00 float               // generate result key              result_value.set(String.valueOf(i)+"\t"+String.valueOf(aver1));              context.write(key, result_value);              System.out.println("result_key=key   "+key.toString());              System.out.println("result_value="+result_value.toString());        }  }  public static void main(String[] args) throws Exception {        Configuration conf = new Configuration();        System.out.println("conf information:  "+conf.toString());        String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();        System.out.println("otherArgs:");        for(int i=0;i<otherArgs.length;i++)            System.out.println(otherArgs[i]);        if (otherArgs.length < 2) {          System.err.println("Usage: wordcount <in> [<in>...] <out>");          System.exit(2);        }        Job job = Job.getInstance(conf, "average holding time");        job.setJarByClass(Hw2Part1.class);        job.setMapperClass(TokenizerMapper.class);        job.setCombinerClass(IntSumCombiner.class);        job.setReducerClass(IntSumReducer.class);        job.setMapOutputKeyClass(Text.class);        job.setMapOutputValueClass(MymulvalueWritable.class);        job.setOutputKeyClass(Text.class);        job.setOutputValueClass(Text.class);        // add the input paths as given by command line        for (int i = 0; i < otherArgs.length - 1; ++i) {          FileInputFormat.addInputPath(job, new Path(otherArgs[i]));          System.out.println(otherArgs[i]);        }        System.out.println(otherArgs[otherArgs.length - 1]);        // add the output path as given by the command line        FileOutputFormat.setOutputPath(job,new Path(otherArgs[otherArgs.length - 1]));        System.exit(job.waitForCompletion(true) ? 0 : 1);   }}
1 0