#270 Add Spark 3.5.0 support, update Scala and Spark dependencies to …

…the latest patches.
AbsaOSS · Oct 25, 2023 · bb4e1e8 · bb4e1e8
1 parent f8b0a6f
commit bb4e1e8
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 16 deletions.
diff --git a/.github/workflows/scala.yml b/.github/workflows/scala.yml
@@ -18,16 +18,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        scala: [2.11.12, 2.12.18, 2.13.11]
-        spark: [2.4.8, 3.2.4, 3.3.2, 3.4.1]
+        scala: [2.11.12, 2.12.18, 2.13.12]
+        spark: [2.4.8, 3.2.4, 3.3.3, 3.4.1]
         exclude:
           - scala: 2.11.12
             spark: 3.2.4
           - scala: 2.11.12
-            spark: 3.3.2
+            spark: 3.3.3
           - scala: 2.11.12
             spark: 3.4.1
-          - scala: 2.13.11
+          - scala: 2.13.12
             spark: 2.4.8
     name: Test Spark ${{matrix.spark}} on Scala ${{matrix.scala}}
     steps:

diff --git a/pramen/build.sbt b/pramen/build.sbt
@@ -21,7 +21,7 @@ import com.github.sbt.jacoco.report.JacocoReportSettings
 
 val scala211 = "2.11.12"
 val scala212 = "2.12.18"
-val scala213 = "2.13.11"
+val scala213 = "2.13.12"
 
 ThisBuild / organization := "za.co.absa.pramen"
 

diff --git a/pramen/project/Versions.scala b/pramen/project/Versions.scala
@@ -18,7 +18,7 @@ import sbt.*
 
 object Versions {
   val defaultSparkVersionForScala211 = "2.4.8"
-  val defaultSparkVersionForScala212 = "3.3.2"
+  val defaultSparkVersionForScala212 = "3.3.3"
   val defaultSparkVersionForScala213 = "3.4.1"
 
   val typesafeConfigVersion = "1.4.0"
@@ -69,17 +69,18 @@ object Versions {
 
   def getDeltaDependency(sparkVersion: String): ModuleID = {
     // According to this: https://docs.delta.io/latest/releases.html
-    val deltaVersion = sparkVersion match {
-      case version if version.startsWith("2.")   => "0.6.1"
-      case version if version.startsWith("3.0.") => "0.8.0"
-      case version if version.startsWith("3.1.") => "1.0.1"
-      case version if version.startsWith("3.2.") => "2.0.1"
-      case version if version.startsWith("3.3.") => "2.2.0"
-      case version if version.startsWith("3.4.") => "2.4.0"
+    val (deltaArtifact, deltaVersion) = sparkVersion match {
+      case version if version.startsWith("2.")   => ("delta-core", "0.6.1")
+      case version if version.startsWith("3.0.") => ("delta-core", "0.8.0")
+      case version if version.startsWith("3.1.") => ("delta-core", "1.0.1")
+      case version if version.startsWith("3.2.") => ("delta-core", "2.0.2")
+      case version if version.startsWith("3.3.") => ("delta-core", "2.2.0")
+      case version if version.startsWith("3.4.") => ("delta-core", "2.4.0")
+      case version if version.startsWith("3.5.") => ("delta-spark", "3.0.0")  // 'delta-core' was renamed to 'delta-spark' since 3.0.0.
       case _                                     => throw new IllegalArgumentException(s"Spark $sparkVersion not supported.")
     }
-    println(s"Using Delta version $deltaVersion")
-    "io.delta" %% "delta-core" % deltaVersion
+    println(s"Using Delta version $deltaArtifact:$deltaVersion")
+    "io.delta" %% deltaArtifact % deltaVersion
   }
 
   def getAbrisDependency(scalaVersion: String): ModuleID = {

diff --git a/pramen/project/build.properties b/pramen/project/build.properties
@@ -13,4 +13,4 @@
 # limitations under the License.
 #
 
-sbt.version=1.9.2
+sbt.version=1.9.7