Merge pull request #78 from jjohannes/generic_imports_resolver

Generalize implementation of ImportsResolver to allow imports from arbitrary npm packages

CHANGELOG.md

@@ -11,7 +11,7 @@ See [Conventional Commits](https://conventionalcommits.org) for commit guideline
 
 ### Features
 
-*
+* allow Solidity imports from arbitrary npm packages [#78](https://github.com/hyperledger-web3j/web3j-solidity-gradle-plugin/pull/78)
 
 ### BREAKING CHANGES
 

src/main/groovy/org/web3j/solidity/gradle/plugin/ImportsResolver.groovy

@@ -12,21 +12,31 @@
  */
 package org.web3j.solidity.gradle.plugin
 
-import groovy.transform.Memoized
+import groovy.io.FileType
+
+import java.util.regex.Pattern
 
 /**
  * Helper class to resolve the external imports from a Solidity file.
  *
- * Supported providers are:
+ * The import resolving is done in three steps:
  * <ul>
- *     <li><a href="https://www.npmjs.com/package/@openzeppelin/contracts">Open Zeppelin</a></li>
- *     <li><a href="https://www.npmjs.com/package/@uniswap/lib">Uniswap</a></li>
+ *     <li>
+ *         First, all packages needed for direct imports are extracted from sol files to generate a package.json.
+ *         This is done in a separate Gradle task, so that the next steps are only performed when direct imports change.
+ *     </li>
+ *     <li>
+ *         Second, required packages are downloaded by npm.
+ *     </li>
+ *     <li>
+ *         Third, sol files that were downloaded are analyzed as well and all packages required are collected.
+ *         This information is used in compileSolidity for the allowed paths and the path remappings.
+ *     </li>
  * </ul>
  */
-@Singleton
 class ImportsResolver {
 
-    private Set<String> PROVIDERS = ["@openzeppelin/contracts", "@uniswap/lib"]
+    private static final IMPORT_PROVIDER_PATTERN = Pattern.compile(".*import.*['\"](@[^/]+/[^/]+).*");
 
     /**
      * Looks for external imports in Solidity files, eg:
@@ -41,17 +51,44 @@ class ImportsResolver {
      * @param nodeProjectDir the Node.js project directory
      * @return
      */
-    @Memoized
-    Map<String, String> resolveImports(final File solFile, final File nodeProjectDir) {
-        final Map<String, String> imports = [:]
-        PROVIDERS.forEach { String provider ->
-            def importFound = !solFile.readLines().findAll {
-                it.contains(provider)
-            }.isEmpty()
+    static Set<String> extractImports(final File solFile) {
+        final Set<String> imports = new TreeSet<>()
+
+        solFile.readLines().each { String line ->
+            final importProviderMatcher = IMPORT_PROVIDER_PATTERN.matcher(line)
+            final importFound = importProviderMatcher.matches()
             if (importFound) {
-                imports.put(provider, "$nodeProjectDir.path/node_modules/$provider")
+                final provider = importProviderMatcher.group(1)
+                imports.add(provider)
             }
         }
+
         return imports
     }
+
+    static Set<String> resolveTransitive(Set<String> directImports, File nodeModulesDir) {
+        final Set<String> allImports = new TreeSet<>()
+        if (directImports.isEmpty()) {
+            return allImports
+        }
+
+        def transitiveResolved = 0
+        allImports.addAll(directImports)
+
+        while (transitiveResolved != allImports.size()) {
+            transitiveResolved = allImports.size()
+            allImports.collect().each { nodeModule ->
+                final packageFolder = new File(nodeModulesDir, nodeModule)
+                if (packageFolder.exists()) { // this may be a dev dependency from a test that we do not need
+                    packageFolder.eachFileRecurse(FileType.FILES) { dependencyFile ->
+                        if (dependencyFile.name.endsWith('.sol')) {
+                            allImports.addAll(extractImports(dependencyFile))
+                        }
+                    }
+                }
+            }
+        }
+
+        return allImports
+    }
 }

src/main/groovy/org/web3j/solidity/gradle/plugin/SolidityCompile.groovy

@@ -12,6 +12,7 @@
  */
 package org.web3j.solidity.gradle.plugin
 
+import org.gradle.api.file.RegularFileProperty
 import org.gradle.api.tasks.*
 import org.web3j.sokt.SolcInstance
 import org.web3j.sokt.SolidityFile
@@ -20,7 +21,7 @@ import org.web3j.sokt.VersionResolver
 import java.nio.file.Paths
 
 @CacheableTask
-class SolidityCompile extends SourceTask {
+abstract class SolidityCompile extends SourceTask {
 
     @Input
     @Optional
@@ -70,8 +71,26 @@ class SolidityCompile extends SourceTask {
     @Optional
     private CombinedOutputComponent[] combinedOutputComponents
 
+    @InputFile
+    @PathSensitive(PathSensitivity.NONE)
+    abstract RegularFileProperty getResolvedImports()
+
+    SolidityCompile() {
+        resolvedImports.convention(project.provider {
+            // Optional file input workaround: https://github.com/gradle/gradle/issues/2016
+            // This is a provider that is only triggered when not overwritten (solidity.resolvePackages = false).
+            def emptyImportsFile = project.layout.buildDirectory.file("sol-imports-empty.txt").get()
+            emptyImportsFile.asFile.parentFile.mkdirs()
+            emptyImportsFile.asFile.createNewFile()
+            return emptyImportsFile
+        })
+    }
+
     @TaskAction
     void compileSolidity() {
+        final imports = resolvedImports.get().asFile.readLines().findAll { !it.isEmpty() }
+        final File nodeModulesDir = project.node.nodeProjectDir.dir("node_modules").get().asFile
+
         for (def contract in source) {
             def options = []
 
@@ -105,18 +124,17 @@ class SolidityCompile extends SourceTask {
                 options.add('--ignore-missing')
             }
 
-            if (!allowPaths.isEmpty()) {
+            if (!allowPaths.isEmpty() || !imports.isEmpty()) {
                 options.add("--allow-paths")
-                options.add(allowPaths.join(','))
+                options.add((allowPaths + imports.collect { new File(nodeModulesDir,it).absolutePath }).join(','))
             }
 
-            final File nodeProjectDir = project.node.nodeProjectDir.asFile.get()
-            def allPathRemappings = pathRemappings + ImportsResolver.instance.resolveImports(contract, nodeProjectDir)
+            pathRemappings.each { key, value ->
+                options.add("$key=$value")
+            }
 
-            if (!allPathRemappings.isEmpty()) {
-                allPathRemappings.forEach { key, value ->
-                    options.add("$key=$value")
-                }
+            imports.each { provider ->
+                options.add("$provider=$nodeModulesDir/$provider")
             }
 
             options.add('--output-dir')

src/main/groovy/org/web3j/solidity/gradle/plugin/SolidityExtractImports.groovy

@@ -0,0 +1,62 @@
+/*
+ * Copyright 2024 Web3 Labs Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.web3j.solidity.gradle.plugin
+
+import groovy.json.JsonBuilder
+import groovy.transform.CompileStatic
+import org.gradle.api.DefaultTask
+import org.gradle.api.file.ConfigurableFileCollection
+import org.gradle.api.file.RegularFileProperty
+import org.gradle.api.provider.Property
+import org.gradle.api.tasks.*
+
+@CacheableTask
+@CompileStatic
+abstract class SolidityExtractImports extends DefaultTask {
+
+    @Input
+    abstract Property<String> getProjectName()
+
+    @InputFiles
+    @PathSensitive(value = PathSensitivity.RELATIVE)
+    @SkipWhenEmpty
+    abstract ConfigurableFileCollection getSources()
+
+    @OutputFile
+    abstract RegularFileProperty getPackageJson()
+
+    SolidityExtractImports() {
+        projectName.convention(project.name)
+    }
+
+    @TaskAction
+    void resolveSolidity() {
+        final Set<String> packages = new TreeSet<>()
+
+        sources.each { contract ->
+            packages.addAll(ImportsResolver.extractImports(contract))
+        }
+
+        final jsonMap = [
+                "name"        : projectName.get(),
+                "description" : "",
+                "repository"  : "",
+                "license"     : "UNLICENSED",
+                "dependencies": packages.collectEntries {
+                    [(it): "latest"]
+                }
+        ]
+
+        packageJson.get().asFile.text = new JsonBuilder(jsonMap).toPrettyString()
+    }
+}

src/main/groovy/org/web3j/solidity/gradle/plugin/SolidityPlugin.groovy

@@ -14,6 +14,7 @@ package org.web3j.solidity.gradle.plugin
 
 import com.github.gradle.node.NodeExtension
 import com.github.gradle.node.NodePlugin
+import com.github.gradle.node.npm.task.NpmInstallTask
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.file.DirectoryProperty
@@ -102,8 +103,7 @@ class SolidityPlugin implements Plugin<Project> {
      */
     private static void configureSolidityCompile(final Project project, final SourceSet sourceSet) {
 
-        def srcSetName = sourceSet.name == 'main' ? '' : capitalize((CharSequence) sourceSet.name)
-        def compileTask = project.tasks.create("compile${srcSetName}Solidity", SolidityCompile)
+        def compileTask = project.tasks.create(sourceSet.getTaskName("compile", "Solidity"), SolidityCompile)
         def soliditySourceSet = sourceSet.convention.plugins[NAME] as SoliditySourceSet
 
         if (!requiresBundledExecutable(project)) {
@@ -146,27 +146,39 @@ class SolidityPlugin implements Plugin<Project> {
         compileTask.outputs.dir(soliditySourceSet.solidity.destinationDirectory)
         compileTask.description = "Compiles $sourceSet.name Solidity source."
 
-        if (project.solidity.resolvePackages) {
-            project.getTasks().named('npmInstall').configure {
-                it.dependsOn(project.getTasks().named("resolveSolidity"))
-            }
-            compileTask.dependsOn(project.getTasks().named("npmInstall"))
-        }
-
         project.getTasks().named('build').configure {
             it.dependsOn(compileTask)
         }
     }
 
     private void configureSolidityResolve(Project target, DirectoryProperty nodeProjectDir) {
-        def resolveSolidity = target.tasks.create("resolveSolidity", SolidityResolve)
-        resolveSolidity.sources = resolvedSolidity.solidity
-        resolveSolidity.description = "Resolve external Solidity contract modules."
-        resolveSolidity.allowPaths = target.solidity.allowPaths
-        resolveSolidity.onlyIf { target.solidity.resolvePackages }
-
-        def packageJson = new File(nodeProjectDir.asFile.get(), "package.json")
-        resolveSolidity.packageJson = packageJson
+
+        if (target.solidity.resolvePackages) {
+            def extractSolidityImports = target.tasks.register("extractSolidityImports", SolidityExtractImports) {
+                it.description = "Extracts imports of external Solidity contract modules."
+                it.sources.from(resolvedSolidity.solidity)
+                it.packageJson.set(nodeProjectDir.file("package.json"))
+            }
+            def npmInstall = target.tasks.named(NpmInstallTask.NAME) {
+                it.dependsOn(extractSolidityImports)
+            }
+            def resolveSolidity = target.tasks.register("resolveSolidity", SolidityResolve) {
+                it.description = "Resolve external Solidity contract modules."
+
+                it.dependsOn(npmInstall)
+                it.packageJson.set(nodeProjectDir.file("package.json"))
+                it.nodeModules.set(nodeProjectDir.dir("node_modules"))
+
+                it.allImports.set(target.layout.buildDirectory.file("sol-imports-all.txt"))
+            }
+
+            final SourceSetContainer sourceSets = target.extensions.getByType(SourceSetContainer.class)
+            sourceSets.all { SourceSet sourceSet ->
+                target.tasks.named(sourceSet.getTaskName("compile", "Solidity"), SolidityCompile) {
+                    it.resolvedImports.set(resolveSolidity.flatMap { it.allImports })
+                }
+            }
+        }
     }
 
     /**