From 8e96e6cd6efdca86f69766e5ad4dd3f8ef7ee103 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Mon, 30 Jun 2025 14:45:50 +0200 Subject: [PATCH 1/5] **Update Scala CI workflows and build configurations** - Refactor `scala-publish.yml` to clarify release and snapshot publishing conditions. - Adjust `docs.yml` trigger to specifically include the `main` branch. - Remove unused Sonatype import from `build.sbt`. - Enhance developer metadata and maintainers list in `build.sbt`. - Update dependencies and assembly configuration to address shading and exclude non-connect classes for the Uber JAR. - Introduce custom POM post-processing for correct dependency scope adjustments. --- .github/workflows/docs.yml | 3 +- .github/workflows/scala-publish.yml | 9 ++++- build.sbt | 57 ++++++++++++++++++++++------- 3 files changed, 53 insertions(+), 16 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d90c3144f..4010c020a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -8,7 +8,8 @@ name: Deploy Docs on: push: - branches: [$default-branch] + branches: + - main workflow_dispatch: permissions: diff --git a/.github/workflows/scala-publish.yml b/.github/workflows/scala-publish.yml index 9da6b377c..e6614d315 100644 --- a/.github/workflows/scala-publish.yml +++ b/.github/workflows/scala-publish.yml @@ -7,6 +7,13 @@ on: tags: - "*.*.*" +# In the case of pushing the tag, it will publish the release. +# In the case of pushing to the main, it will publish the SNAPSHOT + +# From the docs (https://github.com/sbt/sbt-ci-release): +# - git tag pushes are published as regular releases to Maven Central +# - merge into main commits are published as -SNAPSHOT with a unique version number for every commit + jobs: release: strategy: @@ -25,14 +32,12 @@ jobs: fetch-depth: 0 # sbt-git requires the full history to determine the version - name: Set up JDK - if: startsWith(github.ref, 'refs/tags/') uses: actions/setup-java@v4 with: java-version: "${{ matrix.java-version }}" distribution: 'zulu' - name: Run the release of core - if: startsWith(github.ref, 'refs/tags/') run: build/sbt -Dspark.version=${{ matrix.spark-version }} +ci-release "project connect" +ci-release env: PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} diff --git a/build.sbt b/build.sbt index 9787a7cf9..6439a1e2d 100644 --- a/build.sbt +++ b/build.sbt @@ -1,5 +1,3 @@ -import xerial.sbt.Sonatype.sonatypeCentralHost - lazy val sparkVer = sys.props.getOrElse("spark.version", "3.5.5") lazy val sparkMajorVer = sparkVer.substring(0, 1) lazy val sparkBranch = sparkVer.substring(0, 3) @@ -38,6 +36,7 @@ ThisBuild / scmInfo := Some( ScmInfo( url("https://github.com/graphframes/graphframes"), "scm:git@github.com:graphframes/graphframes.git")) +// The list of active maintainers with Write/Maintain/Admin access ThisBuild / developers := List( Developer( id = "rjurney", @@ -48,7 +47,12 @@ ThisBuild / developers := List( id = "SemyonSinchenko", name = "Sem", email = "ssinchenko@apache.org", - url = url("https://github.com/SemyonSinchenko"))) + url = url("https://github.com/SemyonSinchenko")), + Developer( + id = "james-willis", + name = "James Willis", + email = "???", + url = url("https://github.com/james-willis"))) ThisBuild / sonatypeCredentialHost := "s01.oss.sonatype.org" ThisBuild / sonatypeRepository := "https://s01.oss.sonatype.org/service/local" ThisBuild / sonatypeProfileName := "io.graphframes" @@ -113,7 +117,6 @@ lazy val root = (project in file(".")) Global / concurrentRestrictions := Seq(Tags.limitAll(1)), autoAPIMappings := true, coverageHighlighting := false, - Compile / unmanagedSourceDirectories += (Compile / baseDirectory).value / "src" / "main" / s"scala-spark-$sparkMajorVer", // Assembly settings @@ -134,7 +137,7 @@ lazy val root = (project in file(".")) Compile / packageSrc / publishArtifact := true) lazy val connect = (project in file("graphframes-connect")) - .dependsOn(root) + .dependsOn(root % "provided") // It will be provided in the final POM .settings( commonSetting, name := s"graphframes-connect", @@ -152,17 +155,45 @@ lazy val connect = (project in file("graphframes-connect")) assembly / assemblyShadeRules := Seq( ShadeRule.rename("com.google.protobuf.**" -> protobufShadingPattern).inAll), assembly / assemblyMergeStrategy := { - case PathList("google", "protobuf", xs @ _*) => MergeStrategy.discard case PathList("META-INF", xs @ _*) => MergeStrategy.discard case x if x.endsWith("module-info.class") => MergeStrategy.discard - case x => MergeStrategy.first + case _ => MergeStrategy.first }, - assembly / assemblyExcludedJars := (Compile / fullClasspath).value.filter { className => - className.data - .getName() - .contains("scala-library-") || className.data - .getName() - .contains("slf4j-api-") + // I have zero ideas how to apply shading and exclude everything except connect classes in the right way. + // This looks terrible, but it works: + assembly / assemblyExcludedJars := (Compile / fullClasspath).value.filterNot { className => + className.data.getName.contains("GraphFramesConnect") || className.data.getPath.contains( + "graphframes/connect/proto") + }, + // This magic is required to have graphframes-core as a runtime dependency + // at the same time depends on it as provided to avoid packing it to the Uber JAR we are publishing + pomPostProcess := { + val rootModuleName = (root / moduleName).value + node => + import scala.xml._ + import scala.xml.transform._ + + val rewriteRule = new RewriteRule { + override def transform(n: Node): Seq[Node] = n match { + case e: Elem + if e.label == "dependency" && (e \ "artifactId").text.contains(rootModuleName) => + val scope = e \ "scope" + if (scope.text == "provided") { + val children = e.child.filter(_.label != "scope") ++ runtime + Elem( + e.prefix, + e.label, + e.attributes, + e.scope, + minimizeEmpty = false, + children: _*) + } else { + e + } + case _ => n + } + } + new RuleTransformer(rewriteRule).transform(node).head }, publish / skip := false, Compile / packageBin := assembly.value, From 1ba0b2e82891b723bf793a4ca829e74e08c13ccb Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Mon, 30 Jun 2025 20:54:12 +0200 Subject: [PATCH 2/5] Add missing developer email --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 6439a1e2d..68516b5ba 100644 --- a/build.sbt +++ b/build.sbt @@ -51,7 +51,7 @@ ThisBuild / developers := List( Developer( id = "james-willis", name = "James Willis", - email = "???", + email = "jimwillis95@gmail.com", url = url("https://github.com/james-willis"))) ThisBuild / sonatypeCredentialHost := "s01.oss.sonatype.org" ThisBuild / sonatypeRepository := "https://s01.oss.sonatype.org/service/local" From 0975916dd678cdbf6c5ec4665bca34db670b5031 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Wed, 2 Jul 2025 16:44:25 +0200 Subject: [PATCH 3/5] Specify the scope for protobuf-java Added a post-processing to mark protobuf scope to "provided" because it is a part of Apache Spark itself. --- build.sbt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build.sbt b/build.sbt index 68516b5ba..6423724d0 100644 --- a/build.sbt +++ b/build.sbt @@ -190,6 +190,10 @@ lazy val connect = (project in file("graphframes-connect")) } else { e } + case e: Elem + if e.label == "dependency" && (e \ "artifactId").text.contains("protobuf-java") => + val children = e.child ++ provided + Elem(e.prefix, e.label, e.attributes, e.scope, minimizeEmpty = false, children: _*) case _ => n } } From 780d8f7428264b948cf4386b35bf7088fdded68d Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Wed, 2 Jul 2025 18:23:17 +0200 Subject: [PATCH 4/5] Take everything from #617 --- build.sbt | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 925365dfa..76663e31e 100644 --- a/build.sbt +++ b/build.sbt @@ -1,5 +1,3 @@ -import xerial.sbt.Sonatype.sonatypeCentralHost - lazy val sparkVer = sys.props.getOrElse("spark.version", "3.5.5") lazy val sparkMajorVer = sparkVer.substring(0, 1) lazy val sparkBranch = sparkVer.substring(0, 3) @@ -38,6 +36,7 @@ ThisBuild / scmInfo := Some( ScmInfo( url("https://github.com/graphframes/graphframes"), "scm:git@github.com:graphframes/graphframes.git")) +// The list of active maintainers with Write/Maintain/Admin access ThisBuild / developers := List( Developer( id = "rjurney", @@ -48,7 +47,13 @@ ThisBuild / developers := List( id = "SemyonSinchenko", name = "Sem", email = "ssinchenko@apache.org", - url = url("https://github.com/SemyonSinchenko"))) + url = url("https://github.com/SemyonSinchenko")), + Developer( + id = "james-willis", + name = "James Willis", + email = "jimwillis95@gmail.com", + url = url("https://github.com/james-willis")) +) ThisBuild / sonatypeCredentialHost := "s01.oss.sonatype.org" ThisBuild / sonatypeRepository := "https://s01.oss.sonatype.org/service/local" ThisBuild / sonatypeProfileName := "io.graphframes" From e4c62210c07187a34f60236441e8abdc572c2db2 Mon Sep 17 00:00:00 2001 From: semyonsinchenko Date: Wed, 2 Jul 2025 18:28:20 +0200 Subject: [PATCH 5/5] main -> master I always forgot that GF is uses master as a default branch... --- .github/workflows/docs.yml | 2 +- .github/workflows/python-publish.yml | 2 +- .github/workflows/scala-publish.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 4010c020a..b143e707b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -9,7 +9,7 @@ name: Deploy Docs on: push: branches: - - main + - master workflow_dispatch: permissions: diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 7c3dc7f59..7195de3ac 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -3,7 +3,7 @@ name: python-pypi-publish on: push: branches: - - main + - master tags: - "*.*.*" diff --git a/.github/workflows/scala-publish.yml b/.github/workflows/scala-publish.yml index e6614d315..0881d3224 100644 --- a/.github/workflows/scala-publish.yml +++ b/.github/workflows/scala-publish.yml @@ -3,7 +3,7 @@ name: scala-central-publish on: push: branches: - - main + - master tags: - "*.*.*"