[AppleScript] 纯文本查看 复制代码
import org.apache.spark.{SparkConf, SparkContext}
object BigdatBroadCast {
def main(args: Array[String]): Unit = {
val conf=new
SparkConf().setAppName("Bigdata_BroadCast").setMaster("local[2]")
val sc=new SparkContext(conf)
val smallRDD=sc.parallelize(Array(
("1","zhangsan"),
("2","lisi"),
("3","wangwu")
)).collectAsMap()
val smallBroadCast=sc.broadcast(smallRDD)
val bigRDD=sc.parallelize(Array(
("1","school1","male"),
("2","school2","female"),
("3","school3","male"),
("4","school4","male"),
("5","school5","female")
)).map(x=>(x._1,x))
val broadCastValue=smallBroadCast.value
bigRDD.mapPartitions(partitions=>{
for ((key,value)<-partitions
if (broadCastValue.contains(key)))
yield(key,broadCastValue.getOrElse(key,""),value._2,value._3)
}).collect().foreach(println)
/** 结果
* (1,zhangsan,school1,male)
* (2,lisi,school2,female)
* (3,wangwu,school3,male)
*/
}
}