From 8081b772fcd8f3107136e6ad56878404f5ee6124 Mon Sep 17 00:00:00 2001 From: rohithvangapalli Date: Thu, 21 Sep 2023 14:34:27 +0530 Subject: [PATCH 1/2] Updated the almaren and spark versions --- README.md | 18 ++++++++++-------- build.sbt | 4 ++-- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index c538719..81c374a 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,13 @@ For more details check the following [link](https://github.com/GoogleCloudDatapr To add Bigquery Almaren dependency to your sbt build: ``` -libraryDependencies += "com.github.music-of-the-ainur" %% "bigquery-almaren" % "0.0.7-3.2" +libraryDependencies += "com.github.music-of-the-ainur" %% "bigquery-almaren" % "0.0.8-3.2" ``` To run in spark-shell: ``` -spark-shell --master "local[*]" --packages "com.github.music-of-the-ainur:almaren-framework_2.12:0.9.8-3.2,com.github.music-of-the-ainur:bigquery-almaren_2.12:0.0.7-3.2" +spark-shell --master "local[*]" --packages "com.github.music-of-the-ainur:almaren-framework_2.12:0.9.10-3.2,com.github.music-of-the-ainur:bigquery-almaren_2.12:0.0.8-3.2" ``` BigQuery Connector is available in [Maven Central](https://mvnrepository.com/artifact/com.github.music-of-the-ainur) @@ -22,12 +22,14 @@ repository. | versions | Connector Artifact | |----------------------------|-----------------------------------------------------------------| -| Spark 3.3.x and scala 2.13 | `com.github.music-of-the-ainur:bigquery-almaren_2.13:0.0.7-3.3` | -| Spark 3.3.x and scala 2.12 | `com.github.music-of-the-ainur:bigquery-almaren_2.12:0.0.7-3.3` | -| Spark 3.2.x and scala 2.12 | `com.github.music-of-the-ainur:bigquery-almaren_2.12:0.0.7-3.2` | -| Spark 3.1.x and scala 2.12 | `com.github.music-of-the-ainur:bigquery-almaren_2.12:0.0.7-3.1` | -| Spark 2.4.x and scala 2.12 | `com.github.music-of-the-ainur:bigquery-almaren_2.12:0.0.7-2.4` | -| Spark 2.4.x and scala 2.11 | `com.github.music-of-the-ainur:bigquery-almaren_2.11:0.0.7-2.4` | +| Spark 3.4.x and scala 2.13 | `com.github.music-of-the-ainur:bigquery-almaren_2.13:0.0.8-3.4` | +| Spark 3.4.x and scala 2.12 | `com.github.music-of-the-ainur:bigquery-almaren_2.12:0.0.8-3.4` | +| Spark 3.3.x and scala 2.13 | `com.github.music-of-the-ainur:bigquery-almaren_2.13:0.0.8-3.3` | +| Spark 3.3.x and scala 2.12 | `com.github.music-of-the-ainur:bigquery-almaren_2.12:0.0.8-3.3` | +| Spark 3.2.x and scala 2.12 | `com.github.music-of-the-ainur:bigquery-almaren_2.12:0.0.8-3.2` | +| Spark 3.1.x and scala 2.12 | `com.github.music-of-the-ainur:bigquery-almaren_2.12:0.0.8-3.1` | +| Spark 2.4.x and scala 2.12 | `com.github.music-of-the-ainur:bigquery-almaren_2.12:0.0.8-2.4` | +| Spark 2.4.x and scala 2.11 | `com.github.music-of-the-ainur:bigquery-almaren_2.11:0.0.8-2.4` | ## Source and Target diff --git a/build.sbt b/build.sbt index 9da54a1..d2d703d 100644 --- a/build.sbt +++ b/build.sbt @@ -5,7 +5,7 @@ lazy val scala212 = "2.12.10" ThisBuild / scalaVersion := scala212 -val sparkVersion = "3.2.1" +val sparkVersion = "3.2.4" val majorVersionReg = "([0-9]+\\.[0-9]+).{0,}".r val majorVersionReg(majorVersion) = sparkVersion @@ -15,7 +15,7 @@ scalacOptions ++= Seq("-deprecation", "-feature") libraryDependencies ++= Seq( "org.apache.spark" %% "spark-core" % sparkVersion % "provided", "org.apache.spark" %% "spark-sql" % sparkVersion % "provided", - "com.github.music-of-the-ainur" %% "almaren-framework" % s"0.9.8-${majorVersion}" % "provided", + "com.github.music-of-the-ainur" %% "almaren-framework" % s"0.9.10-${majorVersion}" % "provided", "com.google.cloud.spark" %% "spark-bigquery-with-dependencies" % "0.27.0", "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5", "org.scalatest" %% "scalatest" % "3.2.14" % "test" From f98ca2b1ccfab43d1a9520e855bd00730278e707 Mon Sep 17 00:00:00 2001 From: rohithvangapalli Date: Fri, 22 Sep 2023 14:39:22 +0530 Subject: [PATCH 2/2] Updated the bigquery connector version --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index d2d703d..fcbc9af 100644 --- a/build.sbt +++ b/build.sbt @@ -16,7 +16,7 @@ libraryDependencies ++= Seq( "org.apache.spark" %% "spark-core" % sparkVersion % "provided", "org.apache.spark" %% "spark-sql" % sparkVersion % "provided", "com.github.music-of-the-ainur" %% "almaren-framework" % s"0.9.10-${majorVersion}" % "provided", - "com.google.cloud.spark" %% "spark-bigquery-with-dependencies" % "0.27.0", + "com.google.cloud.spark" %% "spark-bigquery-with-dependencies" % "0.32.2", "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5", "org.scalatest" %% "scalatest" % "3.2.14" % "test" )