Spark组件之GraphX学习8--随机图生成和TopK最大入度

更多代码请见：https://github.com/xubo245/SparkLearning

1.解释

前一篇为最大入度，但是有时候需要求topK,这篇博文求的TopK的入度，出度和度的求法类似

2.代码：

/** * @author xubo * ref http://spark.apache.org/docs/1.5.2/graphx-programming-guide.html * time 20160503 */package org.apache.spark.graphx.learningimport org.apache.spark.SparkConfimport org.apache.spark.SparkContextimport org.apache.spark.graphx.Graphimport org.apache.spark.graphx.VertexIdimport org.apache.spark.graphx.VertexRDDimport org.apache.spark.graphx.util.GraphGeneratorsimport org.jets3t.apps.synchronize.Synchronizeimport breeze.linalg.reverseimport breeze.linalg.reverseobject GraphGeneratorsAndTopK {  val K = 3  var arr = new Array[(Int, Int)](K)  def main(args: Array[String]): Unit = {    val conf = new SparkConf().setAppName("GraphGeneratorsAndTopK").setMaster("local[4]")    // Assume the SparkContext has already been constructed    val sc = new SparkContext(conf)    // Import random graph generation library    // Create a graph with "age" as the vertex property.  Here we use a random graph for simplicity.    val graph: Graph[Double, Int] =      GraphGenerators.logNormalGraph(sc, numVertices = 10).mapVertices((id, _) => id.toDouble)    // Compute the number of older followers and their total age    println("Graph:");    println("sc.defaultParallelism:" + sc.defaultParallelism);    println("vertices:");    graph.vertices.collect.foreach(println(_))    println("edges:");    graph.edges.collect.foreach(println(_))    println("count:" + graph.edges.count);    println("\ninDegrees");    graph.inDegrees.foreach(println)    for (i <- 0 until K) {      arr(i) = (0, 0)    }    // Define a reduce operation to compute the highest degree vertex    def max(a: (VertexId, Int), b: (VertexId, Int)): (VertexId, Int) = {      if (a._2 > b._2) a else b    }    // Define a reduce operation to compute the highest degree vertex    def min(a: (VertexId, Int), b: (VertexId, Int)): (VertexId, Int) = {      if (a._2 < b._2) a else b    }    def minInt(a: (Int, Int), b: (Int, Int)): (Int, Int) = {      if (a._2 < b._2) a else b    }    //    arr.reduce(minInt)    println("\ntopK:K=" + K);    def topK(a: (VertexId, Int)): Unit = {      if (a._2 >= arr.reduce(minInt)._2) {        arr = arr.sortBy(_._2).reverse        var tmp = (a._1.toInt, a._2)        var flag = true        for (i <- 0 until arr.length) {          if (a._2 >= arr(i)._2) { //newest max,remove = and last max            if (flag == true) {              for (j <- i + 1 until arr.length reverse) {                arr(j) = arr(j - 1)              }              arr(i) = tmp            }            flag = false          }        }      }    }    graph.inDegrees.foreach(topK(_))    arr.foreach(println)    println("end");  }}

3.结果：

Graph:sc.defaultParallelism:4vertices:(4,4.0)(0,0.0)(8,8.0)(1,1.0)(9,9.0)(5,5.0)(6,6.0)(2,2.0)(3,3.0)(7,7.0)edges:Edge(0,1,1)Edge(0,3,1)Edge(0,6,1)Edge(0,7,1)Edge(0,8,1)Edge(1,2,1)Edge(1,4,1)Edge(1,6,1)Edge(1,8,1)Edge(2,0,1)Edge(2,0,1)Edge(2,1,1)Edge(2,4,1)Edge(2,7,1)Edge(2,8,1)Edge(3,0,1)Edge(3,1,1)Edge(3,2,1)Edge(3,5,1)Edge(3,6,1)Edge(3,7,1)Edge(4,0,1)Edge(4,0,1)Edge(4,3,1)Edge(4,7,1)Edge(5,2,1)Edge(5,4,1)Edge(5,9,1)Edge(6,0,1)Edge(6,0,1)Edge(6,1,1)Edge(6,3,1)Edge(6,3,1)Edge(6,4,1)Edge(6,9,1)Edge(7,2,1)Edge(7,3,1)Edge(7,6,1)Edge(8,2,1)Edge(8,2,1)Edge(8,3,1)Edge(8,4,1)Edge(8,4,1)Edge(8,4,1)Edge(9,5,1)Edge(9,5,1)Edge(9,7,1)count:47inDegrees(1,4)(9,2)(5,3)(3,6)(7,5)(6,4)(2,6)(4,7)(0,7)(8,3)topK:K=3(4,7)(0,7)(3,6)end

参考

【1】 http://spark.apache.org/docs/1.5.2/graphx-programming-guide.html

【2】https://github.com/xubo245/SparkLearning