spark读取mongodb(maven)

来源:互联网 发布:淘宝联盟怎样注册账号 编辑:程序博客网 时间:2024/06/05 04:29
<!--spark连接mongodb-->
<dependency>    <groupId>org.mongodb.spark</groupId>    <artifactId>mongo-spark-connector_2.11</artifactId>    <version>2.0.0</version></dependency><dependency>    <groupId>org.apache.spark</groupId>    <artifactId>spark-sql_2.11</artifactId>    <version>2.0.0</version></dependency>


import com.mongodb.spark._import com.mongodb.spark.config.ReadConfigimport org.apache.spark.{SparkConf, SparkContext}

def main(args: Array[String]): Unit = {    val conf = new SparkConf()    conf.setAppName("Get_Company_From_Mongodb_Spark")            .set("spark.mongodb.input.uri", "mongodb://password:user@mongo_ip:port/app_data.enterprise_data_gov")    val sc = new SparkContext(conf)    val readConfig = ReadConfig(Map("database" -> "app_data", "collection" -> "enterprise_data_gov", "readPreference.name" -> "secondaryPreferred"), Some(ReadConfig(sc)))    val res = sc.loadFromMongoDB(readConfig)}