8，数据挖掘环境搭建-IDEA MAVEN搭建Spark开发环境

IDEA版本：IntelliJ IDEA Community Edition 2016.1.1(64)
IDEA创建Spark项目和pom.xml

1，IDEA 创建Spark项目

1.1，点击IDEA执行文件

1.2 添加scala

1.3 选择Create New Project，选择scala-tools

1.4，输入包名和项目名

1.5，再输入项目名

经过以上步骤，spark开发的环境已经创建完成。但是会出现2个问题，将在下一个篇博客介绍。

2，pom.xml

  <properties>    <maven.compiler.source>1.7</maven.compiler.source>    <maven.compiler.target>1.7</maven.compiler.target>    <encoding>UTF-8</encoding>    <scala.version>2.10.6</scala.version>    <scala.compat.version>2.10.6</scala.compat.version>    <spark.version>1.6.3</spark.version>    <hadoop.version>2.6.4</hadoop.version>  </properties>  <repositories>    <repository>      <id>scala-tools.org</id>      <name>Scala-Tools Maven2 Repository</name>      <url>http://scala-tools.org/repo-releases</url>    </repository>  </repositories>  <pluginRepositories>    <pluginRepository>      <id>scala-tools.org</id>      <name>Scala-Tools Maven2 Repository</name>      <url>http://scala-tools.org/repo-releases</url>    </pluginRepository>  </pluginRepositories>  <dependencies>    <!--scala dependency-->    <dependency>      <groupId>org.scala-lang</groupId>      <artifactId>scala-library</artifactId>      <version>${scala.version}</version>    </dependency>    <!--spark dependency-->    <dependency>      <groupId>org.apache.spark</groupId>      <artifactId>spark-core_2.10</artifactId>      <version>1.6.3</version>    </dependency>    <dependency>      <groupId>org.apache.spark</groupId>      <artifactId>spark-streaming_2.10</artifactId>      <version>1.6.3</version>    </dependency>    <dependency>      <groupId>org.apache.spark</groupId>      <artifactId>spark-sql_2.10</artifactId>      <version>1.6.3</version>    </dependency>    <dependency>      <groupId>org.apache.spark</groupId>      <artifactId>spark-streaming-kafka_2.10</artifactId>      <version>1.6.3</version>    </dependency>    <dependency>      <groupId>org.apache.spark</groupId>      <artifactId>spark-streaming-flume_2.10</artifactId>      <version>1.6.3</version>    </dependency>-->    <!--hadoop dependency-->    <dependency>      <groupId>org.apache.hadoop</groupId>      <artifactId>hadoop-client</artifactId>      <version>2.6.4</version>    </dependency>    <dependency>      <groupId>org.apache.hadoop</groupId>      <artifactId>hadoop-common</artifactId>      <version>2.6.4</version>    </dependency>    <dependency>      <groupId>org.apache.hadoop</groupId>      <artifactId>hadoop-hdfs</artifactId>      <version>2.6.4</version>    </dependency>    <!--system out-->    <dependency>      <groupId>org.specs</groupId>      <artifactId>specs</artifactId>      <version>1.2.5</version>      <scope>test</scope>    </dependency>    <dependency>      <groupId>commons-logging</groupId>      <artifactId>commons-logging</artifactId>      <version>1.1.1</version>      <type>jar</type>    </dependency>    <dependency>      <groupId>org.apache.commons</groupId>      <artifactId>commons-lang3</artifactId>      <version>3.1</version>    </dependency>    <dependency>      <groupId>log4j</groupId>      <artifactId>log4j</artifactId>      <version>1.2.9</version>    </dependency>    <dependency>      <groupId>junit</groupId>      <artifactId>junit</artifactId>      <version>4.12</version>    </dependency>  </dependencies>  <build>    <sourceDirectory>src/main/scala</sourceDirectory>    <testSourceDirectory>src/test/scala</testSourceDirectory>    <plugins>      <plugin>        <groupId>net.alchim31.maven</groupId>        <artifactId>scala-maven-plugin</artifactId>        <version>3.2.0</version>        <executions>          <execution>            <goals>              <goal>compile</goal>              <goal>testCompile</goal>            </goals>            <configuration>              <args>                <arg>-make:transitive</arg>                <arg>-dependencyfile</arg>                <arg>${project.build.directory}/.scala_dependencies</arg>              </args>            </configuration>          </execution>        </executions>      </plugin>      <plugin>        <groupId>org.apache.maven.plugins</groupId>        <artifactId>maven-surefire-plugin</artifactId>        <version>2.18.1</version>        <configuration>          <useFile>false</useFile>          <disableXmlReport>true</disableXmlReport>          <includes>            <include>**/*Test.*</include>            <include>**/*Suite.*</include>          </includes>        </configuration>      </plugin>      <plugin>        <groupId>org.apache.maven.plugins</groupId>        <artifactId>maven-shade-plugin</artifactId>        <version>2.3</version>        <executions>          <execution>            <phase>package</phase>            <goals>              <goal>shade</goal>            </goals>            <configuration>              <filters>                <filter>                  <artifact>*:*</artifact>                  <excludes>                    <exclude>META-INF/*.SF</exclude>                    <exclude>META-INF/*.DSA</exclude>                    <exclude>META-INF/*.RSA</exclude>                  </excludes>                </filter>              </filters>            </configuration>          </execution>        </executions>      </plugin>    </plugins>  </build>  <reporting>    <plugins>      <plugin>        <groupId>org.scala-tools</groupId>        <artifactId>maven-scala-plugin</artifactId>        <configuration>          <scalaVersion>${scala.version}</scalaVersion>        </configuration>      </plugin>    </plugins>  </reporting>