mapreduce实现矩阵相乘

来源：互联网发布：图片制作软件app 编辑：程序博客网时间：2024/04/30 12:26

a的map读取

读取第一个值是1,1,1。它是矩阵a的第一行第一列。那么它要在计算c(1,1) c(1,2)的时候使用（这里c仅仅用2列，假设用n列，那么它的值要在计算c(1,1),c(1,2),c(1,3)…c(1,n)的时候使用）。我们就以 key = 1,1 value = a,1,1 ， key= 1,2 value = a,1,1输出两条数据(1,1) (1,2)是 c(1,1) ,c(1,2)的坐标。

b的map读取

读取第一个值是1,1,10。它是矩阵b的第一行第一列。那么它要在计算c(1,1) c(2,1) c(3,1) c(4,1)的时候使用（这里c仅仅用4行，假设用m行，那么它的值要在计算c(1,1),c(2,1),c(3,1)…c(m,1)的时候使用）。我们就以 key = 1,1 value = b,1,10 , key= 2,1 value = b,1,10 ， key = 3,1 value = b,1,10 , key= 4,1 value = b,1,10输出

3:reduce读取计算

通过mapA和mapB的输出能够得到 key = 1,1 ， value=a,1,1 value=b,1,10 value=a,2,2 value=a,3,3 value = b,3,11l来计算c(1,1)的值

import java.io.IOException;import java.util.Iterator;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IntWritable;import org.apache.hadoop.io.LongWritable;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.lib.input.MultipleInputs;import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public class MartrixMultiply{    /**     * 最后得到的矩阵的列数     */    public static final int COL_COUNT = 2;    /**     * 最后得到的矩阵的行数     */    public static final int ROW_COUNT = 4;    /**     * A矩阵的列数或者是B矩阵的行数     */    public static final int BROW_ACOL= 3;    public static class MartrixMaperA extends Mapper<LongWritable, Text, Text, Text>{        @Override        protected void map(LongWritable key, Text value, Context context)                throws IOException, InterruptedException {            String valueStr = value.toString();            String[] items = valueStr.split(",");            int rowIndex = Integer.parseInt(items[0]);            int colIndex = Integer.parseInt(items[1]);            int valueInt = Integer.parseInt(items[2]);            Text outKey = null;            Text outValue = null;            for(int i=0;i<COL_COUNT;i++){                outKey = new Text(rowIndex + "," + (i+1));                outValue = new Text("a,"+colIndex+","+valueInt);                context.write(outKey, outValue);            }        }    }    public static class MartrixMaperB extends Mapper<LongWritable, Text, Text, Text>{        @Override        protected void map(LongWritable key, Text value, Context context)                throws IOException, InterruptedException {            String valueStr = value.toString();            String[] items = valueStr.split(",");            int rowIndex = Integer.parseInt(items[0]);            int colIndex = Integer.parseInt(items[1]);            int valueInt = Integer.parseInt(items[2]);            Text outKey = null;            Text outValue = null;            for(int i=0;i<ROW_COUNT;i++){                outKey = new Text((i+1) + "," + colIndex);                outValue = new Text("b,"+rowIndex+","+valueInt);                context.write(outKey, outValue);            }        }    }    public static class MartrixReducer extends Reducer<Text,Text,Text,IntWritable>{        @Override        protected void reduce(Text key, Iterable<Text> values,Context context)                throws IOException, InterruptedException {            String[] items = new String[3];            int[] valueA = new int[BROW_ACOL];            int[] valueB = new int[BROW_ACOL];            Iterator<Text> it = values.iterator();            while(it.hasNext()){                items = it.next().toString().split(",");                if(items[0].equals("a")){                    valueA[Integer.parseInt(items[1])-1] = Integer.parseInt(items[2]);                }else if(items[0].equals("b")){                    valueB[Integer.parseInt(items[1])-1] = Integer.parseInt(items[2]);                }            }            int result = 0;            for(int i=0;i<BROW_ACOL;i++){                result += valueA[i]*valueB[i];            }            context.write(key, new IntWritable(result));        }    }    @SuppressWarnings("deprecation")    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {        Path pathA = new Path("hdfs://localhost:9000/Martrix/a.txt");        Path pathB = new Path("hdfs://localhost:9000/Martrix/b.txt");        Path pathOut = new Path("hdfs://localhost:9000/Martrix/out");        Configuration conf = new Configuration();        Job job = new Job(conf,"MartrixMultiply");        job.setJarByClass(MartrixMultiply.class);        MultipleInputs.addInputPath(job, pathA, TextInputFormat.class, MartrixMaperA.class);        MultipleInputs.addInputPath(job, pathB, TextInputFormat.class, MartrixMaperB.class);        job.setReducerClass(MartrixReducer.class);        job.setMapOutputKeyClass(Text.class);        job.setMapOutputValueClass(Text.class);        job.setOutputKeyClass(Text.class);        job.setOutputValueClass(IntWritable.class);        FileOutputFormat.setOutputPath(job, pathOut);        if(job.waitForCompletion(true)){            System.exit(0);        }else{            System.exit(1);        }    }}

0 0