https://www.cnblogs.com/bajiaotai/p/15381309.html
<groupId>com.dxm.sparksql</groupId>
<artifactId>sparksql</artifactId>
<version>1.0-SNAPSHOT</version>
<!-- 指定变量 spark的版本信息 scala的版本信息-->
<properties>
<spark.version>3.0.0</spark.version>
<scala.version>2.12</scala.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core\_${scala.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-yarn\_${scala.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql\_${scala.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.27</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive\_${scala.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
<version>1.2.1</version>
</dependency>
</dependencies>
#根据下面链接 即可查询 spark版本和scala版本的对应关系及依赖配置
https://www.cnblogs.com/bajiaotai/p/16270971.html
println(util.Properties.versionString)
待补充
object TestSparkSQLEnv extends App {
//1.初始化 SparkSession 对象
val spark = SparkSession
.builder
.master("local")
//.appName("SparkSql Entrance Class SparkSession")
//.config("spark.some.config.option", "some-value")
.getOrCreate()
//2.通过 SparkSession 获取 SparkContext
private val sc: SparkContext = spark.sparkContext
//3.设置日志级别
// Valid log levels include: ALL, DEBUG, ERROR, FATAL, INFO, OFF, TRACE, WARN
// This overrides any user-defined log settings //会覆盖掉 用户设置的日志级别 比如 log4j.properties
sc.setLogLevel("ERROR")
import spark.implicits._
//4.创建DataFream
private val rdd2DfByCaseClass: DataFrame = spark.sparkContext
.makeRDD(Array(Person("疫情", "何时"), Person("结束", "呢")))
.toDF("名称", "行动")
rdd2DfByCaseClass.show()
// +----+----+
// |名称|行动|
// +----+----+
// |疫情|何时|
// |结束| 呢|
// +----+----+
//5.关闭资源
spark.stop()
}
如果能正常执行,恭喜你环境搭建没问题,如果遇到问题请留言共同探讨
手机扫一扫
移动阅读更方便
你可能感兴趣的文章