sbt mr-it hadoop 3.3.6

apache · Nov 7, 2023 · 2aa3cb3 · 2aa3cb3
1 parent cb6f0ea
commit 2aa3cb3
Showing 1 changed file with 7 additions and 3 deletions.
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
@@ -34,7 +34,6 @@ object Dependencies {
 
   val zstdJniVersion = sparkClientProjects.map(_.zstdJniVersion).getOrElse("1.5.2-1")
   val lz4JavaVersion = sparkClientProjects.map(_.lz4JavaVersion).getOrElse("1.8.0")
-  val hadoopVersion = mrClientProjects.map(_.hadoopVersion).getOrElse("3.2.4")
 
   // Dependent library versions
   val commonsCompressVersion = "1.4.1"
@@ -44,6 +43,7 @@ object Dependencies {
   val commonsLang3Version = "3.12.0"
   val findbugsVersion = "1.3.9"
   val guavaVersion = "14.0.1"
+  val hadoopVersion = "3.2.4"
   val javaxServletVersion = "3.1.0"
   val junitInterfaceVersion = "0.13.3"
   // don't forget update `junitInterfaceVersion` when we upgrade junit
@@ -900,8 +900,6 @@ trait FlinkClientProjects {
 ////////////////////////////////////////////////////////
 object MRClientProjects {
 
-  val hadoopVersion = "3.3.6"
-
   def mrClient: Project = {
     Project("celeborn-client-mr", file("client-mr/mr"))
       .dependsOn(CelebornCommon.common, CelebornClient.client)
@@ -916,6 +914,8 @@ object MRClientProjects {
   }
 
   def mrIt: Project = {
+    val hadoopVersion = "3.3.6"
+
     Project("celeborn-mr-it", file("tests/mr-it"))
       // ref: https://www.scala-sbt.org/1.x/docs/Multi-Project.html#Classpath+dependencies
       .dependsOn(CelebornCommon.common % "test->test;compile->compile")
@@ -929,6 +929,10 @@ object MRClientProjects {
         libraryDependencies ++= Seq(
           "org.apache.hadoop" % "hadoop-client-minicluster" % hadoopVersion % "test",
           "org.apache.hadoop" % "hadoop-mapreduce-examples" % hadoopVersion % "test",
+          "org.apache.hadoop" % "hadoop-client-api" % hadoopVersion,
+          "org.apache.hadoop" % "hadoop-client-runtime" % hadoopVersion,
+          "org.apache.hadoop" % "hadoop-mapreduce-client-app" % hadoopVersion excludeAll (
+            ExclusionRule("com.google.guava", "guava")),
           "org.bouncycastle" % "bcpkix-jdk15on" % "1.68"
         ) ++ commonUnitTestDependencies
       )