diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0882e66ee3..e29d481510 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,7 +76,7 @@ jobs: uses: actions/setup-java@v4 with: distribution: temurin - java-version: 21 + java-version: 17 - name: Check Documentation run: pnpm nx documentation diff --git a/.nx/workflows/agents.yaml b/.nx/workflows/agents.yaml index 95c493579a..0a0ed37c27 100644 --- a/.nx/workflows/agents.yaml +++ b/.nx/workflows/agents.yaml @@ -64,13 +64,18 @@ launch-templates: - name: Load Cargo Env script: echo "PATH=$HOME/.cargo/bin:$PATH" >> $NX_CLOUD_ENV - - name: Setup Java 21 + - name: Setup Java 17 script: | sudo apt update - sudo apt install -y openjdk-21-jdk - sudo update-alternatives --set java /usr/lib/jvm/java-21-openjdk-amd64/bin/java + sudo apt install -y openjdk-17-jdk + sudo update-alternatives --set java /usr/lib/jvm/java-17-openjdk-amd64/bin/java java -version + - name: Setup Gradle + script: | + ./gradlew wrapper + ./gradlew --version + linux-extra-large: resource-class: 'docker_linux_amd64/extra_large' image: 'ubuntu22.04-node20.11-v10' diff --git a/build.gradle.kts b/build.gradle.kts index dbb3f11d9a..1138e4cfaa 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - id("dev.nx.gradle.project-graph") version("0.0.2") + id("dev.nx.gradle.project-graph") version("0.1.0") id("com.ncorti.ktfmt.gradle") version("+") } diff --git a/docs/generated/manifests/menus.json b/docs/generated/manifests/menus.json index a87b2af32d..669dc313c4 100644 --- a/docs/generated/manifests/menus.json +++ b/docs/generated/manifests/menus.json @@ -8387,6 +8387,23 @@ "isExternal": false, "disableCollapsible": false }, + { + "id": "executors", + "path": "/nx-api/gradle/executors", + "name": "executors", + "children": [ + { + "id": "gradle", + "path": "/nx-api/gradle/executors/gradle", + "name": "gradle", + "children": [], + "isExternal": false, + "disableCollapsible": false + } + ], + "isExternal": false, + "disableCollapsible": false + }, { "id": "generators", "path": "/nx-api/gradle/generators", diff --git a/docs/generated/manifests/nx-api.json b/docs/generated/manifests/nx-api.json index c6aa899dce..b0008b61e4 100644 --- a/docs/generated/manifests/nx-api.json +++ b/docs/generated/manifests/nx-api.json @@ -2103,7 +2103,17 @@ }, "root": "/packages/gradle", "source": "/packages/gradle/src", - "executors": {}, + "executors": { + "/nx-api/gradle/executors/gradle": { + "description": "The Gradlew executor is used to run Gradle tasks.", + "file": "generated/packages/gradle/executors/gradle.json", + "hidden": false, + "name": "gradle", + "originalFilePath": "/packages/gradle/src/executors/gradle/schema.json", + "path": "/nx-api/gradle/executors/gradle", + "type": "executor" + } + }, "generators": { "/nx-api/gradle/generators/init": { "description": "Initializes a Gradle project in the current workspace", diff --git a/docs/generated/packages-metadata.json b/docs/generated/packages-metadata.json index f1b95406f6..f15591a124 100644 --- a/docs/generated/packages-metadata.json +++ b/docs/generated/packages-metadata.json @@ -2087,7 +2087,17 @@ "originalFilePath": "shared/packages/gradle/gradle-plugin" } ], - "executors": [], + "executors": [ + { + "description": "The Gradlew executor is used to run Gradle tasks.", + "file": "generated/packages/gradle/executors/gradle.json", + "hidden": false, + "name": "gradle", + "originalFilePath": "/packages/gradle/src/executors/gradle/schema.json", + "path": "gradle/executors/gradle", + "type": "executor" + } + ], "generators": [ { "description": "Initializes a Gradle project in the current workspace", diff --git a/docs/generated/packages/gradle/executors/gradle.json b/docs/generated/packages/gradle/executors/gradle.json new file mode 100644 index 0000000000..9464088f27 --- /dev/null +++ b/docs/generated/packages/gradle/executors/gradle.json @@ -0,0 +1,37 @@ +{ + "name": "gradle", + "batchImplementation": "./src/executors/gradle/gradle-batch.impl", + "implementation": "/packages/gradle/src/executors/gradle/gradle.impl.ts", + "schema": { + "$schema": "https://json-schema.org/schema", + "version": 2, + "title": "Gradle Impl executor", + "description": "The Gradle Impl executor is used to run Gradle tasks.", + "type": "object", + "properties": { + "taskName": { + "type": "string", + "description": "The name of the Gradle task to run." + }, + "testClassName": { + "type": "string", + "description": "The test class name to run for test task." + }, + "args": { + "oneOf": [ + { "type": "array", "items": { "type": "string" } }, + { "type": "string" } + ], + "description": "The arguments to pass to the Gradle task.", + "examples": [["--warning-mode", "all"], "--stracktrace"] + } + }, + "required": ["taskName"], + "presets": [] + }, + "description": "The Gradlew executor is used to run Gradle tasks.", + "aliases": [], + "hidden": false, + "path": "/packages/gradle/src/executors/gradle/schema.json", + "type": "executor" +} diff --git a/docs/shared/reference/sitemap.md b/docs/shared/reference/sitemap.md index 1121e6f542..b860956ce8 100644 --- a/docs/shared/reference/sitemap.md +++ b/docs/shared/reference/sitemap.md @@ -485,6 +485,8 @@ - [gradle](/nx-api/gradle) - [documents](/nx-api/gradle/documents) - [Overview](/nx-api/gradle/documents/overview) + - [executors](/nx-api/gradle/executors) + - [gradle](/nx-api/gradle/executors/gradle) - [generators](/nx-api/gradle/generators) - [init](/nx-api/gradle/generators/init) - [ci-workflow](/nx-api/gradle/generators/ci-workflow) diff --git a/e2e/gradle/src/gradle.test.ts b/e2e/gradle/src/gradle.test.ts index 8f6786b5c8..e11a2fc37f 100644 --- a/e2e/gradle/src/gradle.test.ts +++ b/e2e/gradle/src/gradle.test.ts @@ -31,9 +31,7 @@ describe('Gradle', () => { expect(projects).toContain(gradleProjectName); const buildOutput = runCLI('build app', { verbose: true }); - expect(buildOutput).toContain('nx run list:'); expect(buildOutput).toContain(':list:classes'); - expect(buildOutput).toContain('nx run utilities:'); expect(buildOutput).toContain(':utilities:classes'); checkFilesExist( @@ -83,11 +81,8 @@ dependencies { let buildOutput = runCLI('build app2', { verbose: true }); // app2 depends on app - expect(buildOutput).toContain('nx run app:'); expect(buildOutput).toContain(':app:classes'); - expect(buildOutput).toContain('nx run list:'); expect(buildOutput).toContain(':list:classes'); - expect(buildOutput).toContain('nx run utilities:'); expect(buildOutput).toContain(':utilities:classes'); checkFilesExist(`app2/build/libs/app2.jar`); @@ -96,7 +91,7 @@ dependencies { it('should run atomized test target', () => { updateJson('nx.json', (json) => { json.plugins.find((p) => p.plugin === '@nx/gradle').options[ - 'ciTargetName' + 'ciTestTargetName' ] = 'test-ci'; return json; }); diff --git a/packages/gradle/batch-runner/.gitignore b/packages/gradle/batch-runner/.gitignore new file mode 100644 index 0000000000..667ce03db0 --- /dev/null +++ b/packages/gradle/batch-runner/.gitignore @@ -0,0 +1,3 @@ +# Ignore Gradle project-specific cache directory +bin +build \ No newline at end of file diff --git a/packages/gradle/batch-runner/build.gradle.kts b/packages/gradle/batch-runner/build.gradle.kts new file mode 100644 index 0000000000..7d311e98fc --- /dev/null +++ b/packages/gradle/batch-runner/build.gradle.kts @@ -0,0 +1,33 @@ +group = "dev.nx.gradle" + +plugins { + // Apply the org.jetbrains.kotlin.jvm Plugin to add support for Kotlin. + id("org.jetbrains.kotlin.jvm") version "2.1.10" + // Apply the application plugin to add support for building a CLI application in Java. + application + id("com.github.johnrengelman.shadow") version "8.1.1" + id("com.ncorti.ktfmt.gradle") version "+" + id("dev.nx.gradle.project-graph") version "0.1.0" +} + +repositories { + // Use Maven Central for resolving dependencies. + mavenCentral() + // need for gradle-tooling-api + maven { url = uri("https://repo.gradle.org/gradle/libs-releases/") } +} + +dependencies { + val toolingApiVersion = "8.13" // Match the Gradle version you're working with + + implementation("org.gradle:gradle-tooling-api:$toolingApiVersion") + runtimeOnly("org.slf4j:slf4j-simple:1.7.10") + implementation("com.google.code.gson:gson:2.10.1") +} + +application { + // Define the main class for the application. + mainClass.set("dev.nx.gradle.NxBatchRunnerKt") +} + +kotlin { jvmToolchain { languageVersion.set(JavaLanguageVersion.of(17)) } } diff --git a/packages/gradle/batch-runner/project.json b/packages/gradle/batch-runner/project.json new file mode 100644 index 0000000000..337e70d6f6 --- /dev/null +++ b/packages/gradle/batch-runner/project.json @@ -0,0 +1,33 @@ +{ + "name": "gradle-batch-runner", + "$schema": "node_modules/nx/schemas/project-schema.json", + "projectRoot": "packages/gradle/batch-runner", + "sourceRoot": "packages/gradle/batch-runner/src", + "targets": { + "assemble": { + "command": "./gradlew :batch-runner:assemble", + "inputs": [ + "{projectRoot}/src/**", + "{projectRoot}/build.gradle.kts", + "{projectRoot}/settings.gradle.kts" + ], + "outputs": ["{projectRoot}/build"], + "cache": true + }, + "test": { + "command": "./gradlew :batch-runner:test", + "options": { + "args": [] + }, + "cache": true + }, + "lint": { + "command": "./gradlew :batch-runner:ktfmtCheck", + "cache": true + }, + "format": { + "command": "./gradlew :batch-runner:ktfmtFormat", + "cache": true + } + } +} diff --git a/packages/gradle/batch-runner/settings.gradle.kts b/packages/gradle/batch-runner/settings.gradle.kts new file mode 100644 index 0000000000..d0639301fd --- /dev/null +++ b/packages/gradle/batch-runner/settings.gradle.kts @@ -0,0 +1,17 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * The settings file is used to specify which projects to include in your build. + * For more detailed information on multi-project builds, please refer to https://docs.gradle.org/8.5/userguide/building_swift_projects.html in the Gradle documentation. + * This project uses @Incubating APIs which are subject to change. + */ + +pluginManagement { + repositories { + mavenLocal() + mavenCentral() + gradlePluginPortal() + } +} + +rootProject.name = "batch-runner" diff --git a/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/NxBatchRunner.kt b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/NxBatchRunner.kt new file mode 100644 index 0000000000..7b55ece743 --- /dev/null +++ b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/NxBatchRunner.kt @@ -0,0 +1,51 @@ +package dev.nx.gradle + +import com.google.gson.Gson +import dev.nx.gradle.cli.configureLogger +import dev.nx.gradle.cli.parseArgs +import dev.nx.gradle.runner.runTasksInParallel +import dev.nx.gradle.util.logger +import java.io.File +import kotlin.system.exitProcess +import org.gradle.tooling.GradleConnector +import org.gradle.tooling.ProjectConnection + +fun main(args: Array) { + val options = parseArgs(args) + configureLogger(options.quiet) + + if (options.workspaceRoot.isBlank()) { + logger.severe("โŒ Missing required arguments --workspaceRoot") + exitProcess(1) + } + if (options.tasks.isEmpty()) { + logger.severe("โŒ Missing required arguments --tasks") + exitProcess(1) + } + + var connection: ProjectConnection? = null + + try { + connection = + GradleConnector.newConnector().forProjectDirectory(File(options.workspaceRoot)).connect() + + val results = runTasksInParallel(connection, options.tasks, options.args) + + val reportJson = Gson().toJson(results) + println(reportJson) + + val summary = results.values.groupBy { it.success } + logger.info( + "๐Ÿ“Š Summary: โœ… ${summary[true]?.size ?: 0} succeeded, โŒ ${summary[false]?.size ?: 0} failed") + } catch (e: Exception) { + logger.severe("๐Ÿ’ฅ Failed to run tasks: ${e.message}") + exitProcess(1) + } finally { + try { + connection?.close() + logger.info("โœ… Gradle connection closed.") + } catch (e: Exception) { + logger.warning("โš ๏ธ Failed to close Gradle connection cleanly: ${e.message}") + } + } +} diff --git a/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/cli/ArgParser.kt b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/cli/ArgParser.kt new file mode 100644 index 0000000000..c11363076a --- /dev/null +++ b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/cli/ArgParser.kt @@ -0,0 +1,47 @@ +package dev.nx.gradle.cli + +import com.google.gson.Gson +import com.google.gson.reflect.TypeToken +import dev.nx.gradle.data.GradleTask +import dev.nx.gradle.data.NxBatchOptions +import dev.nx.gradle.util.logger + +fun parseArgs(args: Array): NxBatchOptions { + val argMap = mutableMapOf() + + args.forEach { + when { + it.startsWith("--") && it.contains("=") -> { + val (key, value) = it.split("=", limit = 2) + argMap[key] = value + } + it.startsWith("--") -> { + argMap[it] = "true" + } + } + } + + val gson = Gson() + val tasksJson = argMap["--tasks"] + val tasksMap: Map = + if (tasksJson != null) { + val taskType = object : TypeToken>() {}.type + gson.fromJson(tasksJson, taskType) + } else emptyMap() + + return NxBatchOptions( + workspaceRoot = argMap["--workspaceRoot"] ?: "", + tasks = tasksMap, + args = argMap["--args"] ?: "", + quiet = argMap["--quiet"]?.toBoolean() ?: false) +} + +fun configureLogger(quiet: Boolean) { + if (quiet) { + logger.setLevel(java.util.logging.Level.OFF) + logger.useParentHandlers = false + logger.handlers.forEach { it.level = java.util.logging.Level.OFF } + } else { + logger.setLevel(java.util.logging.Level.INFO) + } +} diff --git a/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/data/GradlewTask.kt b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/data/GradlewTask.kt new file mode 100644 index 0000000000..13d100033a --- /dev/null +++ b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/data/GradlewTask.kt @@ -0,0 +1,3 @@ +package dev.nx.gradle.data + +data class GradleTask(val taskName: String, val testClassName: String? = null) diff --git a/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/data/NxBatchOptions.kt b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/data/NxBatchOptions.kt new file mode 100644 index 0000000000..7589ae2268 --- /dev/null +++ b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/data/NxBatchOptions.kt @@ -0,0 +1,8 @@ +package dev.nx.gradle.data + +data class NxBatchOptions( + val workspaceRoot: String, + val tasks: Map, + val args: String, + val quiet: Boolean +) diff --git a/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/data/TaskResult.kt b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/data/TaskResult.kt new file mode 100644 index 0000000000..c37de97765 --- /dev/null +++ b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/data/TaskResult.kt @@ -0,0 +1,8 @@ +package dev.nx.gradle.data + +data class TaskResult( + val success: Boolean, + val startTime: Long, + val endTime: Long, + var terminalOutput: String +) diff --git a/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/runner/BuildListener.kt b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/runner/BuildListener.kt new file mode 100644 index 0000000000..488cca7edf --- /dev/null +++ b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/runner/BuildListener.kt @@ -0,0 +1,53 @@ +package dev.nx.gradle.runner + +import dev.nx.gradle.data.GradleTask +import dev.nx.gradle.data.TaskResult +import dev.nx.gradle.util.logger +import kotlin.math.max +import kotlin.math.min +import org.gradle.tooling.events.ProgressEvent +import org.gradle.tooling.events.task.TaskFailureResult +import org.gradle.tooling.events.task.TaskFinishEvent +import org.gradle.tooling.events.task.TaskStartEvent +import org.gradle.tooling.events.task.TaskSuccessResult + +fun buildListener( + tasks: Map, + taskStartTimes: MutableMap, + taskResults: MutableMap +): (ProgressEvent) -> Unit = { event -> + when (event) { + is TaskStartEvent -> { + tasks.entries + .find { it.value.taskName == event.descriptor.taskPath } + ?.key + ?.let { nxTaskId -> + taskStartTimes[nxTaskId] = min(System.currentTimeMillis(), event.eventTime) + } + } + is TaskFinishEvent -> { + val taskPath = event.descriptor.taskPath + val success = + when (event.result) { + is TaskSuccessResult -> { + logger.info("โœ… Task finished successfully: $taskPath") + true + } + is TaskFailureResult -> { + logger.warning("โŒ Task failed: $taskPath") + false + } + else -> true + } + + tasks.entries + .find { it.value.taskName == taskPath } + ?.key + ?.let { nxTaskId -> + val endTime = max(System.currentTimeMillis(), event.eventTime) + val startTime = taskStartTimes[nxTaskId] ?: event.result.startTime + taskResults[nxTaskId] = TaskResult(success, startTime, endTime, "") + } + } + } +} diff --git a/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/runner/GradleRunner.kt b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/runner/GradleRunner.kt new file mode 100644 index 0000000000..a692a1e235 --- /dev/null +++ b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/runner/GradleRunner.kt @@ -0,0 +1,175 @@ +package dev.nx.gradle.runner + +import dev.nx.gradle.data.GradleTask +import dev.nx.gradle.data.TaskResult +import dev.nx.gradle.runner.OutputProcessor.buildTerminalOutput +import dev.nx.gradle.runner.OutputProcessor.splitOutputPerTask +import dev.nx.gradle.util.logger +import java.io.ByteArrayOutputStream +import org.gradle.tooling.ProjectConnection +import org.gradle.tooling.events.OperationType + +fun runTasksInParallel( + connection: ProjectConnection, + tasks: Map, + additionalArgs: String, +): Map { + logger.info("โ–ถ๏ธ Running all tasks in a single Gradle run: ${tasks.keys.joinToString(", ")}") + + val (testClassTasks, buildTasks) = tasks.entries.partition { it.value.testClassName != null } + + logger.info("๐Ÿงช Test launcher tasks: ${testClassTasks.joinToString(", ") { it.key }}") + logger.info("๐Ÿ› ๏ธ Build launcher tasks: ${buildTasks.joinToString(", ") { it.key }}") + + val allResults = mutableMapOf() + + val outputStream = ByteArrayOutputStream() + val errorStream = ByteArrayOutputStream() + + val args = buildList { + addAll(listOf("--info", "--continue", "--parallel", "--build-cache")) + addAll(additionalArgs.split(" ").filter { it.isNotBlank() }) + } + + val taskNames = tasks.values.map { it.taskName }.distinct() + + if (buildTasks.isNotEmpty()) { + allResults.putAll( + runBuildLauncher( + connection, + buildTasks.associate { it.key to it.value }, + taskNames, + args, + outputStream, + errorStream)) + } + + if (testClassTasks.isNotEmpty()) { + allResults.putAll( + runTestLauncher( + connection, + testClassTasks.associate { it.key to it.value }, + taskNames, + args, + outputStream, + errorStream)) + } + + return allResults +} + +fun runBuildLauncher( + connection: ProjectConnection, + tasks: Map, + taskNames: List, + args: List, + outputStream: ByteArrayOutputStream, + errorStream: ByteArrayOutputStream +): Map { + val taskStartTimes = mutableMapOf() + val taskResults = mutableMapOf() + + var globalOutput: String + + try { + connection + .newBuild() + .apply { + forTasks(*taskNames.toTypedArray()) + withArguments(*args.toTypedArray()) + setStandardOutput(outputStream) + setStandardError(errorStream) + addProgressListener(buildListener(tasks, taskStartTimes, taskResults), OperationType.TASK) + } + .run() + globalOutput = buildTerminalOutput(outputStream, errorStream) + } catch (e: Exception) { + globalOutput = + buildTerminalOutput(outputStream, errorStream) + "\nException occurred: ${e.message}" + logger.warning("\ud83d\udca5 Gradle run failed: ${e.message}") + } finally { + outputStream.close() + errorStream.close() + } + + val perTaskOutput = splitOutputPerTask(globalOutput) + tasks.forEach { (taskId, taskConfig) -> + val taskOutput = perTaskOutput[taskConfig.taskName] ?: globalOutput + taskResults[taskId]?.let { taskResults[taskId] = it.copy(terminalOutput = taskOutput) } + } + + logger.info("\u2705 Finished build tasks") + return taskResults +} + +fun runTestLauncher( + connection: ProjectConnection, + tasks: Map, + taskNames: List, + args: List, + outputStream: ByteArrayOutputStream, + errorStream: ByteArrayOutputStream +): Map { + val taskStartTimes = mutableMapOf() + val taskResults = mutableMapOf() + val testTaskStatus = mutableMapOf() + val testStartTimes = mutableMapOf() + val testEndTimes = mutableMapOf() + + tasks.forEach { (nxTaskId, taskConfig) -> + if (taskConfig.testClassName != null) { + testTaskStatus[nxTaskId] = true + } + } + + val globalStart = System.currentTimeMillis() + var globalOutput: String + + try { + connection + .newTestLauncher() + .apply { + forTasks(*taskNames.toTypedArray()) + tasks.values.mapNotNull { it.testClassName }.forEach { withJvmTestClasses(it) } + withArguments(*args.toTypedArray()) + setStandardOutput(outputStream) + setStandardError(errorStream) + addProgressListener( + testListener( + tasks, taskStartTimes, taskResults, testTaskStatus, testStartTimes, testEndTimes), + OperationType.TEST) + } + .run() + globalOutput = buildTerminalOutput(outputStream, errorStream) + } catch (e: Exception) { + globalOutput = + buildTerminalOutput(outputStream, errorStream) + "\nException occurred: ${e.message}" + logger.warning("\ud83d\udca5 Gradle test run failed: ${e.message}") + } finally { + outputStream.close() + errorStream.close() + } + + val globalEnd = System.currentTimeMillis() + + tasks.forEach { (nxTaskId, taskConfig) -> + if (taskConfig.testClassName != null) { + val success = testTaskStatus[nxTaskId] ?: false + val startTime = testStartTimes[nxTaskId] ?: globalStart + val endTime = testEndTimes[nxTaskId] ?: globalEnd + + if (!taskResults.containsKey(nxTaskId)) { + taskResults[nxTaskId] = TaskResult(success, startTime, endTime, "") + } + } + } + + val perTaskOutput = splitOutputPerTask(globalOutput) + tasks.forEach { (taskId, taskConfig) -> + val taskOutput = perTaskOutput[taskConfig.taskName] ?: globalOutput + taskResults[taskId]?.let { taskResults[taskId] = it.copy(terminalOutput = taskOutput) } + } + + logger.info("\u2705 Finished test tasks") + return taskResults +} diff --git a/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/runner/OutputProcessor.kt b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/runner/OutputProcessor.kt new file mode 100644 index 0000000000..9e9459fe8b --- /dev/null +++ b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/runner/OutputProcessor.kt @@ -0,0 +1,33 @@ +package dev.nx.gradle.runner + +import java.io.ByteArrayOutputStream + +object OutputProcessor { + fun buildTerminalOutput(stdOut: ByteArrayOutputStream, stdErr: ByteArrayOutputStream): String { + val output = stdOut.toString("UTF-8") + val errorOutput = stdErr.toString("UTF-8") + return buildString { + if (output.isNotBlank()) append(output).append("\n") + if (errorOutput.isNotBlank()) append(errorOutput) + } + } + + fun splitOutputPerTask(globalOutput: String): Map { + val unescapedOutput = globalOutput.replace("\\u003e", ">").replace("\\n", "\n") + val taskHeaderRegex = Regex("(?=> Task (:[^\\s]+))") + val sections = unescapedOutput.split(taskHeaderRegex) + val taskOutputMap = mutableMapOf() + + for (section in sections) { + val lines = section.trim().lines() + if (lines.isEmpty()) continue + val header = lines.firstOrNull { it.startsWith("> Task ") } + if (header != null) { + val taskMatch = Regex("> Task (:[^\\s]+)").find(header) + val taskName = taskMatch?.groupValues?.get(1) ?: continue + taskOutputMap[taskName] = section.trim() + } + } + return taskOutputMap + } +} diff --git a/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/runner/TestListener.kt b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/runner/TestListener.kt new file mode 100644 index 0000000000..72fe90ec75 --- /dev/null +++ b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/runner/TestListener.kt @@ -0,0 +1,59 @@ +package dev.nx.gradle.runner + +import dev.nx.gradle.data.GradleTask +import dev.nx.gradle.data.TaskResult +import dev.nx.gradle.util.logger +import kotlin.math.max +import kotlin.math.min +import org.gradle.tooling.events.ProgressEvent +import org.gradle.tooling.events.task.TaskFinishEvent +import org.gradle.tooling.events.task.TaskStartEvent +import org.gradle.tooling.events.test.* + +fun testListener( + tasks: Map, + taskStartTimes: MutableMap, + taskResults: MutableMap, + testTaskStatus: MutableMap, + testStartTimes: MutableMap, + testEndTimes: MutableMap +): (ProgressEvent) -> Unit = { event -> + when (event) { + is TaskStartEvent, + is TaskFinishEvent -> buildListener(tasks, taskStartTimes, taskResults)(event) + is TestStartEvent -> { + (event.descriptor as? JvmTestOperationDescriptor)?.className?.let { className -> + tasks.entries + .find { entry -> entry.value.testClassName?.let { className.endsWith(it) } ?: false } + ?.key + ?.let { nxTaskId -> + testStartTimes.compute(nxTaskId) { _, old -> + min(old ?: event.eventTime, event.eventTime) + } + } + } + } + is TestFinishEvent -> { + (event.descriptor as? JvmTestOperationDescriptor)?.className?.let { className -> + tasks.entries + .find { entry -> entry.value.testClassName?.let { className.endsWith(it) } ?: false } + ?.key + ?.let { nxTaskId -> + testEndTimes.compute(nxTaskId) { _, old -> + max(old ?: event.eventTime, event.eventTime) + } + when (event.result) { + is TestSuccessResult -> logger.info("\u2705 Test passed: $nxTaskId $className") + is TestFailureResult -> { + testTaskStatus[nxTaskId] = false + logger.warning("\u274C Test failed: $nxTaskId $className") + } + is TestSkippedResult -> + logger.warning("\u26A0\uFE0F Test skipped: $nxTaskId $className") + else -> logger.warning("\u26A0\uFE0F Unknown test result: $nxTaskId $className") + } + } + } + } + } +} diff --git a/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/util/Logger.kt b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/util/Logger.kt new file mode 100644 index 0000000000..b3b7d54b1e --- /dev/null +++ b/packages/gradle/batch-runner/src/main/kotlin/dev/nx/gradle/util/Logger.kt @@ -0,0 +1,5 @@ +package dev.nx.gradle.util + +import java.util.logging.Logger + +val logger: Logger = Logger.getLogger("NxBatchRunner") diff --git a/packages/gradle/executors.json b/packages/gradle/executors.json index 3a64718e93..23af203201 100644 --- a/packages/gradle/executors.json +++ b/packages/gradle/executors.json @@ -1,3 +1,10 @@ { - "executors": {} + "executors": { + "gradle": { + "batchImplementation": "./src/executors/gradle/gradle-batch.impl", + "implementation": "./src/executors/gradle/gradle.impl", + "schema": "./src/executors/gradle/schema.json", + "description": "The Gradlew executor is used to run Gradle tasks." + } + } } diff --git a/packages/gradle/project-graph/build.gradle.kts b/packages/gradle/project-graph/build.gradle.kts index 7b81d2f463..14e31f7dfb 100644 --- a/packages/gradle/project-graph/build.gradle.kts +++ b/packages/gradle/project-graph/build.gradle.kts @@ -3,14 +3,14 @@ plugins { `maven-publish` signing id("com.ncorti.ktfmt.gradle") version "+" - id("dev.nx.gradle.project-graph") version "0.0.2" + id("dev.nx.gradle.project-graph") version "0.1.0" id("org.jetbrains.kotlin.jvm") version "2.1.10" id("com.gradle.plugin-publish") version "1.2.1" } group = "dev.nx.gradle" -version = "0.0.2" +version = "0.1.0" repositories { mavenCentral() } @@ -118,3 +118,5 @@ signing { } tasks.test { useJUnitPlatform() } + +java { toolchain.languageVersion.set(JavaLanguageVersion.of(17)) } diff --git a/packages/gradle/project-graph/src/main/kotlin/dev/nx/gradle/utils/CiTargetsUtils.kt b/packages/gradle/project-graph/src/main/kotlin/dev/nx/gradle/utils/CiTargetsUtils.kt index c6a9a8d00b..3fc0704cbc 100644 --- a/packages/gradle/project-graph/src/main/kotlin/dev/nx/gradle/utils/CiTargetsUtils.kt +++ b/packages/gradle/project-graph/src/main/kotlin/dev/nx/gradle/utils/CiTargetsUtils.kt @@ -1,90 +1,86 @@ package dev.nx.gradle.utils -import dev.nx.gradle.data.* +import dev.nx.gradle.data.NxTargets +import dev.nx.gradle.data.TargetGroups import java.io.File import org.gradle.api.Task import org.gradle.api.file.FileCollection const val testCiTargetGroup = "verification" -/** - * Add atomized ci test targets Going to loop through each test files and create a target for each - * It is going to modify targets and targetGroups in place - */ +private val testFileNameRegex = + Regex("^(?!(abstract|fake)).*?(Test)(s)?\\d*", RegexOption.IGNORE_CASE) + +private val classDeclarationRegex = Regex("""class\s+([A-Za-z_][A-Za-z0-9_]*)""") + fun addTestCiTargets( testFiles: FileCollection, projectBuildPath: String, testTask: Task, + testTargetName: String, targets: NxTargets, targetGroups: TargetGroups, projectRoot: String, workspaceRoot: String, - ciTargetName: String + ciTestTargetName: String ) { ensureTargetGroupExists(targetGroups, testCiTargetGroup) - val gradlewCommand = getGradlewCommand() val ciDependsOn = mutableListOf>() - val filteredTestFiles = testFiles.filter { isTestFile(it, workspaceRoot) } + testFiles + .filter { isTestFile(it, workspaceRoot) } + .forEach { testFile -> + val className = getTestClassNameIfAnnotated(testFile) ?: return@forEach - filteredTestFiles.forEach { testFile -> - val className = getTestClassNameIfAnnotated(testFile) ?: return@forEach + val targetName = "$ciTestTargetName--$className" + targets[targetName] = + buildTestCiTarget( + projectBuildPath, className, testFile, testTask, projectRoot, workspaceRoot) + targetGroups[testCiTargetGroup]?.add(targetName) - val testCiTarget = - buildTestCiTarget( - gradlewCommand = gradlewCommand, - projectBuildPath = projectBuildPath, - testClassName = className, - testFile = testFile, - testTask = testTask, - projectRoot = projectRoot, - workspaceRoot = workspaceRoot) + ciDependsOn.add(mapOf("target" to targetName, "projects" to "self", "params" to "forward")) + } - val targetName = "$ciTargetName--$className" - targets[targetName] = testCiTarget - targetGroups[testCiTargetGroup]?.add(targetName) - - ciDependsOn.add(mapOf("target" to targetName, "projects" to "self", "params" to "forward")) - } - - testTask.logger.info("$testTask ci tasks: $ciDependsOn") + testTask.logger.info("${testTask.path} generated CI targets: ${ciDependsOn.map { it["target"] }}") if (ciDependsOn.isNotEmpty()) { ensureParentCiTarget( - targets = targets, - targetGroups = targetGroups, - ciTargetName = ciTargetName, - projectBuildPath = projectBuildPath, - dependsOn = ciDependsOn) + targets, + targetGroups, + ciTestTargetName, + projectBuildPath, + testTask, + testTargetName, + ciDependsOn) } } private fun getTestClassNameIfAnnotated(file: File): String? { - if (!file.exists()) return null - - val content = file.readText() - if (!content.contains("@Test")) return null - - val classRegex = Regex("""class\s+([A-Za-z_][A-Za-z0-9_]*)""") - val match = classRegex.find(content) - return match?.groupValues?.get(1) + return file + .takeIf { it.exists() } + ?.readText() + ?.takeIf { it.contains("@Test") || it.contains("@TestTemplate") } + ?.let { content -> + val className = classDeclarationRegex.find(content)?.groupValues?.getOrNull(1) + return if (className != null && !className.startsWith("Fake")) { + className + } else { + null + } + } } fun ensureTargetGroupExists(targetGroups: TargetGroups, group: String) { - if (!targetGroups.containsKey(group)) { - targetGroups[group] = mutableListOf() - } + targetGroups.getOrPut(group) { mutableListOf() } } private fun isTestFile(file: File, workspaceRoot: String): Boolean { val fileName = file.name.substringBefore(".") - val regex = "^(?!abstract).*?(Test)(s)?\\d*".toRegex(RegexOption.IGNORE_CASE) - return file.path.startsWith(workspaceRoot) && regex.matches(fileName) + return file.path.startsWith(workspaceRoot) && testFileNameRegex.matches(fileName) } private fun buildTestCiTarget( - gradlewCommand: String, projectBuildPath: String, testClassName: String, testFile: File, @@ -94,7 +90,11 @@ private fun buildTestCiTarget( ): MutableMap { val target = mutableMapOf( - "command" to "$gradlewCommand ${projectBuildPath}:test --tests $testClassName", + "executor" to "@nx/gradle:gradle", + "options" to + mapOf( + "taskName" to "${projectBuildPath}:${testTask.name}", + "testClassName" to testClassName), "metadata" to getMetadata("Runs Gradle test $testClassName in CI", projectBuildPath, "test"), "cache" to true, @@ -103,14 +103,14 @@ private fun buildTestCiTarget( getDependsOnForTask(testTask, null) ?.takeIf { it.isNotEmpty() } ?.let { - testTask.logger.info("$testTask: processed ${it.size} dependsOn") + testTask.logger.info("${testTask.path}: found ${it.size} dependsOn entries") target["dependsOn"] = it } getOutputsForTask(testTask, projectRoot, workspaceRoot) ?.takeIf { it.isNotEmpty() } ?.let { - testTask.logger.info("$testTask: processed ${it.size} outputs") + testTask.logger.info("${testTask.path}: found ${it.size} outputs entries") target["outputs"] = it } @@ -120,24 +120,31 @@ private fun buildTestCiTarget( private fun ensureParentCiTarget( targets: NxTargets, targetGroups: TargetGroups, - ciTargetName: String, + ciTestTargetName: String, projectBuildPath: String, + testTask: Task, + testTargetName: String, dependsOn: List> ) { val ciTarget = - targets.getOrPut(ciTargetName) { - mutableMapOf().apply { - put("executor", "nx:noop") - put("metadata", getMetadata("Runs Gradle Tests in CI", projectBuildPath, "test", "test")) - put("dependsOn", mutableListOf>()) - put("cache", true) - } + targets.getOrPut(ciTestTargetName) { + mutableMapOf( + "executor" to "nx:noop", + "metadata" to + getMetadata( + "Runs Gradle ${testTask.name} in CI", + projectBuildPath, + testTask.name, + testTargetName), + "dependsOn" to mutableListOf(), + "cache" to true) } - val dependsOnList = ciTarget.getOrPut("dependsOn") { mutableListOf() } as MutableList + @Suppress("UNCHECKED_CAST") + val dependsOnList = ciTarget["dependsOn"] as? MutableList ?: mutableListOf() dependsOnList.addAll(dependsOn) - if (targetGroups[testCiTargetGroup]?.contains(ciTargetName) != true) { - targetGroups[testCiTargetGroup]?.add(ciTargetName) + if (!targetGroups[testCiTargetGroup].orEmpty().contains(ciTestTargetName)) { + targetGroups[testCiTargetGroup]?.add(ciTestTargetName) } } diff --git a/packages/gradle/project-graph/src/main/kotlin/dev/nx/gradle/utils/projectDependencyUtils.kt b/packages/gradle/project-graph/src/main/kotlin/dev/nx/gradle/utils/ProjectDependencyUtils.kt similarity index 100% rename from packages/gradle/project-graph/src/main/kotlin/dev/nx/gradle/utils/projectDependencyUtils.kt rename to packages/gradle/project-graph/src/main/kotlin/dev/nx/gradle/utils/ProjectDependencyUtils.kt diff --git a/packages/gradle/project-graph/src/main/kotlin/dev/nx/gradle/utils/ProjectUtils.kt b/packages/gradle/project-graph/src/main/kotlin/dev/nx/gradle/utils/ProjectUtils.kt index 5984a1ac33..61977c7902 100644 --- a/packages/gradle/project-graph/src/main/kotlin/dev/nx/gradle/utils/ProjectUtils.kt +++ b/packages/gradle/project-graph/src/main/kotlin/dev/nx/gradle/utils/ProjectUtils.kt @@ -63,41 +63,41 @@ fun processTargetsForProject( workspaceRoot: String, cwd: String ): GradleTargets { - val targets: NxTargets = mutableMapOf>() - val targetGroups: TargetGroups = mutableMapOf>() + val targets: NxTargets = mutableMapOf() + val targetGroups: TargetGroups = mutableMapOf() val externalNodes = mutableMapOf() + val projectRoot = project.projectDir.path - project.logger.info("Using workspace root $workspaceRoot") - - var projectBuildPath: String = - project - .buildTreePath // get the build path of project e.g. :app, :utils:number-utils, :buildSrc - if (projectBuildPath.endsWith(":")) { // root project is ":", manually remove last : - projectBuildPath = projectBuildPath.dropLast(1) - } - val logger = project.logger - logger.info("${Date()} ${project}: process targets") + logger.info("Using workspace root: $workspaceRoot") - var gradleProject = project.buildTreePath - if (!gradleProject.endsWith(":")) { - gradleProject += ":" - } + val projectBuildPath = project.buildTreePath.trimEnd(':') + + logger.info("${Date()} ${project}: Process targets") + + val ciTestTargetName = targetNameOverrides["ciTestTargetName"] + val ciIntTestTargetName = targetNameOverrides["ciIntTestTargetName"] + val ciCheckTargetName = targetNameOverrides.getOrDefault("ciCheckTargetName", "check-ci") + val testTargetName = targetNameOverrides.getOrDefault("testTargetName", "test") + val intTestTargetName = targetNameOverrides.getOrDefault("intTestTargetName", "intTest") + + val testTasks = project.getTasksByName("test", false) + val intTestTasks = project.getTasksByName("intTest", false) + val hasCiTestTarget = ciTestTargetName != null && testTasks.isNotEmpty() + val hasCiIntTestTarget = ciIntTestTargetName != null && intTestTasks.isNotEmpty() project.tasks.forEach { task -> try { - logger.info("${Date()} ${project.name}: Processing $task") - val taskName = targetNameOverrides.getOrDefault(task.name + "TargetName", task.name) - // add task to target groups - val group: String? = task.group - if (!group.isNullOrBlank()) { - if (targetGroups.contains(group)) { - targetGroups[group]?.add(task.name) - } else { - targetGroups[group] = mutableListOf(task.name) - } - } + val now = Date() + logger.info("$now ${project.name}: Processing task ${task.path}") + + val taskName = targetNameOverrides.getOrDefault("${task.name}TargetName", task.name) + + // Group task under its group if available + task.group + ?.takeIf { it.isNotBlank() } + ?.let { group -> targetGroups.getOrPut(group) { mutableListOf() }.add(taskName) } val target = processTask( @@ -105,53 +105,63 @@ fun processTargetsForProject( projectBuildPath, projectRoot, workspaceRoot, - cwd, externalNodes, dependencies, targetNameOverrides) + targets[taskName] = target - val ciTargetName = targetNameOverrides.getOrDefault("ciTargetName", null) - ciTargetName?.let { - if (task.name.startsWith("compileTest")) { - val testTask = project.getTasksByName("test", false) - if (testTask.isNotEmpty()) { - addTestCiTargets( - task.inputs.sourceFiles, - projectBuildPath, - testTask.first(), - targets, - targetGroups, - projectRoot, - workspaceRoot, - it) - } - } - - // Add the `$ciTargetName-check` target when processing the "check" task - if (task.name == "check") { - val replacedDependencies = - (target["dependsOn"] as? List<*>)?.map { dep -> - if (dep.toString() == targetNameOverrides.getOrDefault("testTargetName", "test")) - ciTargetName - else dep.toString() - } ?: emptyList() - - // Copy the original target and override "dependsOn" - val newTarget = target.toMutableMap() - newTarget["dependsOn"] = replacedDependencies - - val ciCheckTargetName = "$ciTargetName-check" - targets[ciCheckTargetName] = newTarget - - ensureTargetGroupExists(targetGroups, testCiTargetGroup) - targetGroups[testCiTargetGroup]?.add(ciCheckTargetName) - } + if (hasCiTestTarget && task.name.startsWith("compileTest")) { + addTestCiTargets( + task.inputs.sourceFiles, + projectBuildPath, + testTasks.first(), + testTargetName, + targets, + targetGroups, + projectRoot, + workspaceRoot, + ciTestTargetName!!) } - logger.info("${Date()} ${project.name}: Processed $task") + + if (hasCiIntTestTarget && task.name.startsWith("compileIntTest")) { + addTestCiTargets( + task.inputs.sourceFiles, + projectBuildPath, + intTestTasks.first(), + intTestTargetName, + targets, + targetGroups, + projectRoot, + workspaceRoot, + ciIntTestTargetName!!) + } + + if (task.name == "check" && (hasCiTestTarget || hasCiIntTestTarget)) { + val replacedDependencies = + (target["dependsOn"] as? List<*>)?.map { dep -> + when (dep.toString()) { + testTargetName -> ciTestTargetName ?: dep + intTestTargetName -> ciIntTestTargetName ?: dep + else -> dep + }.toString() + } ?: emptyList() + + val newTarget: MutableMap = + mutableMapOf( + "dependsOn" to replacedDependencies, + "executor" to "nx:noop", + "cache" to true, + "metadata" to getMetadata("Runs Gradle Check in CI", projectBuildPath, "check")) + + targets[ciCheckTargetName] = newTarget + ensureTargetGroupExists(targetGroups, testCiTargetGroup) + targetGroups[testCiTargetGroup]?.add(ciCheckTargetName) + } + + logger.info("$now ${project.name}: Processed task ${task.path}") } catch (e: Exception) { - logger.info("${task}: process task error $e") - logger.debug("Stack trace:", e) + logger.error("Error processing task ${task.path}: ${e.message}", e) } } diff --git a/packages/gradle/project-graph/src/main/kotlin/dev/nx/gradle/utils/TaskUtils.kt b/packages/gradle/project-graph/src/main/kotlin/dev/nx/gradle/utils/TaskUtils.kt index ac49eab006..ed33b6c33d 100644 --- a/packages/gradle/project-graph/src/main/kotlin/dev/nx/gradle/utils/TaskUtils.kt +++ b/packages/gradle/project-graph/src/main/kotlin/dev/nx/gradle/utils/TaskUtils.kt @@ -1,10 +1,9 @@ package dev.nx.gradle.utils -import dev.nx.gradle.data.* -import org.gradle.api.Named -import org.gradle.api.NamedDomainObjectProvider +import dev.nx.gradle.data.Dependency +import dev.nx.gradle.data.ExternalDepData +import dev.nx.gradle.data.ExternalNode import org.gradle.api.Task -import org.gradle.api.tasks.TaskProvider /** * Process a task and convert it into target Going to populate: @@ -13,14 +12,12 @@ import org.gradle.api.tasks.TaskProvider * - outputs * - command * - metadata - * - options with cwd and args */ fun processTask( task: Task, projectBuildPath: String, projectRoot: String, workspaceRoot: String, - cwd: String, externalNodes: MutableMap, dependencies: MutableSet, targetNameOverrides: Map @@ -51,13 +48,14 @@ fun processTask( target["dependsOn"] = dependsOn } - val gradlewCommand = getGradlewCommand() - target["command"] = "$gradlewCommand ${projectBuildPath}:${task.name}" + target["executor"] = "@nx/gradle:gradle" - val metadata = getMetadata(task.description ?: "Run ${task.name}", projectBuildPath, task.name) + val metadata = + getMetadata( + task.description ?: "Run ${projectBuildPath}.${task.name}", projectBuildPath, task.name) target["metadata"] = metadata - target["options"] = mapOf("cwd" to cwd) + target["options"] = mapOf("taskName" to "${projectBuildPath}:${task.name}") return target } @@ -190,54 +188,9 @@ fun getDependsOnForTask( } return try { - val dependsOnEntries = task.dependsOn - - // Prefer task.dependsOn - if (dependsOnEntries.isNotEmpty()) { - val resolvedTasks = - dependsOnEntries.flatMap { dep -> - when (dep) { - is Task -> listOf(dep) - - is TaskProvider<*>, - is NamedDomainObjectProvider<*> -> { - val providerName = (dep as Named).name - val foundTask = task.project.tasks.findByName(providerName) - if (foundTask != null) { - listOf(foundTask) - } else { - task.logger.info( - "${dep::class.simpleName} '$providerName' did not resolve to a task in project ${task.project.name}") - emptyList() - } - } - - is String -> { - val foundTask = task.project.tasks.findByPath(dep) - if (foundTask != null) { - listOf(foundTask) - } else { - task.logger.info( - "Task string '$dep' could not be resolved in project ${task.project.name}") - emptyList() - } - } - - else -> { - task.logger.info( - "Unhandled dependency type ${dep::class.java} for task ${task.path}") - emptyList() - } - } - } - - if (resolvedTasks.isNotEmpty()) { - return mapTasksToNames(resolvedTasks) - } - } - - // Fallback: taskDependencies.getDependencies(task) - val fallbackDeps = + // get depends on using taskDependencies.getDependencies(task) because task.dependsOn has + // missing deps + val dependsOn = try { task.taskDependencies.getDependencies(null) } catch (e: Exception) { @@ -246,8 +199,8 @@ fun getDependsOnForTask( emptySet() } - if (fallbackDeps.isNotEmpty()) { - return mapTasksToNames(fallbackDeps) + if (dependsOn.isNotEmpty()) { + return mapTasksToNames(dependsOn) } null @@ -266,14 +219,15 @@ fun getDependsOnForTask( fun getMetadata( description: String?, projectBuildPath: String, - taskName: String, + helpTaskName: String, nonAtomizedTarget: String? = null ): Map { val gradlewCommand = getGradlewCommand() return mapOf( "description" to description, "technologies" to arrayOf("gradle"), - "help" to mapOf("command" to "$gradlewCommand help --task ${projectBuildPath}:${taskName}"), + "help" to + mapOf("command" to "$gradlewCommand help --task ${projectBuildPath}:${helpTaskName}"), "nonAtomizedTarget" to nonAtomizedTarget) } diff --git a/packages/gradle/project-graph/src/test/kotlin/dev/nx/gradle/utils/AddTestCiTargetsTest.kt b/packages/gradle/project-graph/src/test/kotlin/dev/nx/gradle/utils/AddTestCiTargetsTest.kt index 58d41d8561..ce8b5bcdf8 100644 --- a/packages/gradle/project-graph/src/test/kotlin/dev/nx/gradle/utils/AddTestCiTargetsTest.kt +++ b/packages/gradle/project-graph/src/test/kotlin/dev/nx/gradle/utils/AddTestCiTargetsTest.kt @@ -44,17 +44,18 @@ class AddTestCiTargetsTest { val targets = mutableMapOf>() val targetGroups = mutableMapOf>() - val ciTargetName = "ci" + val ciTestTargetName = "ci" addTestCiTargets( testFiles = testFiles, projectBuildPath = ":project-a", testTask = testTask, + testTargetName = "test", targets = targets, targetGroups = targetGroups, projectRoot = projectRoot.absolutePath, workspaceRoot = workspaceRoot.absolutePath, - ciTargetName = ciTargetName) + ciTestTargetName = ciTestTargetName) // Assert each test file created a CI target assertTrue(targets.containsKey("ci--MyFirstTest")) @@ -73,7 +74,7 @@ class AddTestCiTargetsTest { assertEquals(2, dependsOn!!.size) val firstTarget = targets["ci--MyFirstTest"]!! - assertTrue(firstTarget["command"].toString().contains("--tests MyFirstTest")) + assertEquals(firstTarget["executor"], "@nx/gradle:gradle") assertEquals(true, firstTarget["cache"]) assertTrue((firstTarget["inputs"] as Array<*>)[0].toString().contains("{projectRoot}")) assertEquals("nx:noop", parentCi["executor"]) diff --git a/packages/gradle/project-graph/src/test/kotlin/dev/nx/gradle/utils/CreateNodeForProjectTest.kt b/packages/gradle/project-graph/src/test/kotlin/dev/nx/gradle/utils/CreateNodeForProjectTest.kt index 49289e86b1..9aa36e763a 100644 --- a/packages/gradle/project-graph/src/test/kotlin/dev/nx/gradle/utils/CreateNodeForProjectTest.kt +++ b/packages/gradle/project-graph/src/test/kotlin/dev/nx/gradle/utils/CreateNodeForProjectTest.kt @@ -45,7 +45,6 @@ class CreateNodeForProjectTest { assertEquals(project.name, projectNode.name) assertNotNull(projectNode.targets["compileJava"], "Expected compileJava target") assertNotNull(projectNode.targets["test"], "Expected test target") - assertEquals("build", projectNode.metadata.targetGroups.keys.firstOrNull()) // Dependencies and external nodes should default to empty assertTrue(result.dependencies.isEmpty(), "Expected no dependencies") diff --git a/packages/gradle/project-graph/src/test/kotlin/dev/nx/gradle/utils/ProcessTaskUtilsTest.kt b/packages/gradle/project-graph/src/test/kotlin/dev/nx/gradle/utils/ProcessTaskUtilsTest.kt index 994a92234e..b6309390dc 100644 --- a/packages/gradle/project-graph/src/test/kotlin/dev/nx/gradle/utils/ProcessTaskUtilsTest.kt +++ b/packages/gradle/project-graph/src/test/kotlin/dev/nx/gradle/utils/ProcessTaskUtilsTest.kt @@ -83,13 +83,12 @@ class ProcessTaskUtilsTest { projectBuildPath = ":project", projectRoot = project.projectDir.path, workspaceRoot = project.rootDir.path, - cwd = ".", externalNodes = mutableMapOf(), dependencies = mutableSetOf(), targetNameOverrides = emptyMap()) assertEquals(true, result["cache"]) - assertTrue((result["command"] as String).contains("gradlew")) + assertEquals(result["executor"], "@nx/gradle:gradle") assertNotNull(result["metadata"]) assertNotNull(result["options"]) } diff --git a/packages/gradle/project.json b/packages/gradle/project.json index 34899a4f56..cb5b865e9b 100644 --- a/packages/gradle/project.json +++ b/packages/gradle/project.json @@ -12,6 +12,11 @@ } }, "build-base": { + "dependsOn": [ + "^build-base", + "build-native", + "gradle-batch-runner:assemble" + ], "executor": "@nx/js:tsc", "options": { "assets": [ @@ -42,6 +47,11 @@ "glob": "**/*.d.ts", "output": "/" }, + { + "input": "packages/gradle/batch-runner", + "glob": "**/*.jar", + "output": "/batch-runner" + }, { "input": "", "glob": "LICENSE", diff --git a/packages/gradle/src/executors/gradle/gradle-batch.impl.ts b/packages/gradle/src/executors/gradle/gradle-batch.impl.ts new file mode 100644 index 0000000000..5995b65359 --- /dev/null +++ b/packages/gradle/src/executors/gradle/gradle-batch.impl.ts @@ -0,0 +1,103 @@ +import { ExecutorContext, output, TaskGraph } from '@nx/devkit'; +import { + LARGE_BUFFER, + RunCommandsOptions, +} from 'nx/src/executors/run-commands/run-commands.impl'; +import { BatchResults } from 'nx/src/tasks-runner/batch/batch-messages'; +import { gradleExecutorSchema } from './schema'; +import { findGradlewFile } from '../../utils/exec-gradle'; +import { dirname, join } from 'path'; +import { execSync } from 'child_process'; + +export const batchRunnerPath = join( + __dirname, + '../../../batch-runner/build/libs/batch-runner-all.jar' +); + +interface GradleTask { + taskName: string; + testClassName: string; +} + +export default async function gradleBatch( + taskGraph: TaskGraph, + inputs: Record, + overrides: RunCommandsOptions, + context: ExecutorContext +): Promise { + try { + const projectName = taskGraph.tasks[taskGraph.roots[0]]?.target?.project; + let projectRoot = context.projectGraph.nodes[projectName]?.data?.root ?? ''; + const gradlewPath = findGradlewFile(join(projectRoot, 'project.json')); // find gradlew near project root + const root = join(context.root, dirname(gradlewPath)); + + // set args with passed in args and overrides in command line + const input = inputs[taskGraph.roots[0]]; + + let args = + typeof input.args === 'string' + ? input.args.trim().split(' ') + : Array.isArray(input.args) + ? input.args + : []; + if (overrides.__overrides_unparsed__.length) { + args.push(...overrides.__overrides_unparsed__); + } + + const gradlewTasksToRun: Record = Object.entries( + taskGraph.tasks + ).reduce((gradlewTasksToRun, [taskId, task]) => { + const gradlewTaskName = inputs[task.id].taskName; + const testClassName = inputs[task.id].testClassName; + gradlewTasksToRun[taskId] = { + taskName: gradlewTaskName, + testClassName: testClassName, + }; + return gradlewTasksToRun; + }, {}); + const gradlewBatchStart = performance.mark(`gradlew-batch:start`); + const batchResults = execSync( + `java -jar ${batchRunnerPath} --tasks='${JSON.stringify( + gradlewTasksToRun + )}' --workspaceRoot=${root} --args='${args + .join(' ') + .replaceAll("'", '"')}' ${ + process.env.NX_VERBOSE_LOGGING === 'true' ? '' : '--quiet' + }`, + { + windowsHide: true, + env: process.env, + maxBuffer: LARGE_BUFFER, + } + ); + const gradlewBatchEnd = performance.mark(`gradlew-batch:end`); + performance.measure( + `gradlew-batch`, + gradlewBatchStart.name, + gradlewBatchEnd.name + ); + const gradlewBatchResults = JSON.parse( + batchResults.toString() + ) as BatchResults; + + Object.keys(taskGraph.tasks).forEach((taskId) => { + if (!gradlewBatchResults[taskId]) { + gradlewBatchResults[taskId] = { + success: false, + terminalOutput: `Gradlew batch failed`, + }; + } + }); + + return gradlewBatchResults; + } catch (e) { + output.error({ + title: `Gradlew batch failed`, + bodyLines: [e.toString()], + }); + return taskGraph.roots.reduce((acc, key) => { + acc[key] = { success: false, terminalOutput: e.toString() }; + return acc; + }, {} as BatchResults); + } +} diff --git a/packages/gradle/src/executors/gradle/gradle.impl.ts b/packages/gradle/src/executors/gradle/gradle.impl.ts new file mode 100644 index 0000000000..b1503812a3 --- /dev/null +++ b/packages/gradle/src/executors/gradle/gradle.impl.ts @@ -0,0 +1,39 @@ +import { ExecutorContext } from '@nx/devkit'; +import { gradleExecutorSchema } from './schema'; +import { findGradlewFile } from '../../utils/exec-gradle'; +import { dirname, join } from 'node:path'; +import runCommandsImpl from 'nx/src/executors/run-commands/run-commands.impl'; + +export default async function gradleExecutor( + options: gradleExecutorSchema, + context: ExecutorContext +): Promise<{ success: boolean }> { + let projectRoot = + context.projectGraph.nodes[context.projectName]?.data?.root ?? context.root; + let gradlewPath = findGradlewFile(join(projectRoot, 'project.json')); // find gradlew near project root + gradlewPath = join(context.root, gradlewPath); + + let args = + typeof options.args === 'string' + ? options.args.trim().split(' ') + : Array.isArray(options.args) + ? options.args + : []; + if (options.testClassName) { + args.push(`--tests`, options.testClassName); + } + try { + await runCommandsImpl( + { + command: `${gradlewPath} ${options.taskName}`, + cwd: dirname(gradlewPath), + args: args, + __unparsed__: [], + }, + context + ); + return { success: true }; + } catch (e) { + return { success: false }; + } +} diff --git a/packages/gradle/src/executors/gradle/schema.d.ts b/packages/gradle/src/executors/gradle/schema.d.ts new file mode 100644 index 0000000000..75ce6ee6b1 --- /dev/null +++ b/packages/gradle/src/executors/gradle/schema.d.ts @@ -0,0 +1,5 @@ +export interface gradleExecutorSchema { + taskName: string; + testClassName?: string; + args?: string[] | string; +} diff --git a/packages/gradle/src/executors/gradle/schema.json b/packages/gradle/src/executors/gradle/schema.json new file mode 100644 index 0000000000..2c1cf0693b --- /dev/null +++ b/packages/gradle/src/executors/gradle/schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/schema", + "version": 2, + "title": "Gradle Impl executor", + "description": "The Gradle Impl executor is used to run Gradle tasks.", + "type": "object", + "properties": { + "taskName": { + "type": "string", + "description": "The name of the Gradle task to run." + }, + "testClassName": { + "type": "string", + "description": "The test class name to run for test task." + }, + "args": { + "oneOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string" + } + ], + "description": "The arguments to pass to the Gradle task.", + "examples": [["--warning-mode", "all"], "--stracktrace"] + } + }, + "required": ["taskName"] +} diff --git a/packages/gradle/src/plugin-v1/utils/get-project-report-lines.ts b/packages/gradle/src/plugin-v1/utils/get-project-report-lines.ts index 78d434c589..34c3f7ab10 100644 --- a/packages/gradle/src/plugin-v1/utils/get-project-report-lines.ts +++ b/packages/gradle/src/plugin-v1/utils/get-project-report-lines.ts @@ -1,7 +1,5 @@ import { AggregateCreateNodesError, logger, output } from '@nx/devkit'; import { execGradleAsync, newLineSeparator } from '../../utils/exec-gradle'; -import { existsSync } from 'fs'; -import { dirname, join } from 'path'; /** * This function executes the gradle projectReportAll task and returns the output as an array of lines. diff --git a/packages/gradle/src/plugin/__snapshots__/nodes.spec.ts.snap b/packages/gradle/src/plugin/__snapshots__/nodes.spec.ts.snap index 620e9490da..9887bc8202 100644 --- a/packages/gradle/src/plugin/__snapshots__/nodes.spec.ts.snap +++ b/packages/gradle/src/plugin/__snapshots__/nodes.spec.ts.snap @@ -84,7 +84,7 @@ exports[`@nx/gradle/plugin/nodes should create nodes based on gradle for nested ] `; -exports[`@nx/gradle/plugin/nodes should create nodes with atomized tests targets based on gradle if ciTargetName is specified 1`] = ` +exports[`@nx/gradle/plugin/nodes should create nodes with atomized tests targets based on gradle if ciTestTargetName is specified 1`] = ` [ [ "proj/application/build.gradle", @@ -466,7 +466,7 @@ exports[`@nx/gradle/plugin/nodes should create nodes with atomized tests targets ] `; -exports[`@nx/gradle/plugin/nodes should not create nodes with atomized tests targets based on gradle if ciTargetName is not specified 1`] = ` +exports[`@nx/gradle/plugin/nodes should not create nodes with atomized tests targets based on gradle if ciTestTargetName is not specified 1`] = ` [ [ "proj/application/build.gradle", diff --git a/packages/gradle/src/plugin/dependencies.ts b/packages/gradle/src/plugin/dependencies.ts index 65b2e21673..9fb42992d7 100644 --- a/packages/gradle/src/plugin/dependencies.ts +++ b/packages/gradle/src/plugin/dependencies.ts @@ -8,7 +8,7 @@ import { validateDependency, workspaceRoot, } from '@nx/devkit'; -import { relative } from 'node:path'; +import { join, relative } from 'node:path'; import { getCurrentProjectGraphReport, @@ -29,7 +29,11 @@ export const createDependencies: CreateDependencies< Array.from(GRALDEW_FILES) ); const { gradlewFiles } = splitConfigFiles(files); - await populateProjectGraph(context.workspaceRoot, gradlewFiles, options); + await populateProjectGraph( + context.workspaceRoot, + gradlewFiles.map((file) => join(workspaceRoot, file)), + options + ); const { dependencies: dependenciesFromReport } = getCurrentProjectGraphReport(); diff --git a/packages/gradle/src/plugin/nodes.spec.ts b/packages/gradle/src/plugin/nodes.spec.ts index 31bee1d8cb..56809e3042 100644 --- a/packages/gradle/src/plugin/nodes.spec.ts +++ b/packages/gradle/src/plugin/nodes.spec.ts @@ -82,12 +82,12 @@ describe('@nx/gradle/plugin/nodes', () => { expect(results).toMatchSnapshot(); }); - it('should create nodes with atomized tests targets based on gradle if ciTargetName is specified', async () => { + it('should create nodes with atomized tests targets based on gradle if ciTestTargetName is specified', async () => { const results = await createNodesFunction( ['proj/application/build.gradle'], { buildTargetName: 'build', - ciTargetName: 'test-ci', + ciTestTargetName: 'test-ci', }, context ); @@ -95,7 +95,7 @@ describe('@nx/gradle/plugin/nodes', () => { expect(results).toMatchSnapshot(); }); - it('should not create nodes with atomized tests targets based on gradle if ciTargetName is not specified', async () => { + it('should not create nodes with atomized tests targets based on gradle if ciTestTargetName is not specified', async () => { const results = await createNodesFunction( ['proj/application/build.gradle'], { diff --git a/packages/gradle/src/plugin/nodes.ts b/packages/gradle/src/plugin/nodes.ts index e5f58519ae..b4a2b07a90 100644 --- a/packages/gradle/src/plugin/nodes.ts +++ b/packages/gradle/src/plugin/nodes.ts @@ -80,6 +80,12 @@ export const makeCreateNodesForGradleConfigFile = options: GradlePluginOptions | undefined, context: CreateNodesContext ) => { + if (process.env.VERCEL) { + // Vercel does not allow JAVA_VERSION to be set + // skip on Vercel + return {}; + } + const projectRoot = dirname(gradleFilePath); options = normalizeOptions(options); diff --git a/packages/gradle/src/plugin/utils/gradle-plugin-options.ts b/packages/gradle/src/plugin/utils/gradle-plugin-options.ts index 267f521423..e0d6515644 100644 --- a/packages/gradle/src/plugin/utils/gradle-plugin-options.ts +++ b/packages/gradle/src/plugin/utils/gradle-plugin-options.ts @@ -1,6 +1,7 @@ export interface GradlePluginOptions { testTargetName?: string; - ciTargetName?: string; + ciTestTargetName?: string; + ciIntTestTargetName?: string; [taskTargetName: string]: string | undefined | boolean; } diff --git a/packages/gradle/src/utils/versions.ts b/packages/gradle/src/utils/versions.ts index 5de3983d9b..0d9880a83f 100644 --- a/packages/gradle/src/utils/versions.ts +++ b/packages/gradle/src/utils/versions.ts @@ -1,4 +1,4 @@ export const nxVersion = require('../../package.json').version; export const gradleProjectGraphPluginName = 'dev.nx.gradle.project-graph'; -export const gradleProjectGraphVersion = '0.0.2'; +export const gradleProjectGraphVersion = '0.1.0'; diff --git a/packages/nx/src/tasks-runner/batch/run-batch.ts b/packages/nx/src/tasks-runner/batch/run-batch.ts index 6f39ef6fef..3c272c87c2 100644 --- a/packages/nx/src/tasks-runner/batch/run-batch.ts +++ b/packages/nx/src/tasks-runner/batch/run-batch.ts @@ -75,7 +75,7 @@ async function runTasks( const results = await batchExecutor.batchImplementationFactory()( batchTaskGraph, input, - tasks[0].overrides, + tasks[tasks.length - 1].overrides, context ); diff --git a/packages/nx/src/tasks-runner/task-orchestrator.ts b/packages/nx/src/tasks-runner/task-orchestrator.ts index 75a8f13f82..2ee7e21f40 100644 --- a/packages/nx/src/tasks-runner/task-orchestrator.ts +++ b/packages/nx/src/tasks-runner/task-orchestrator.ts @@ -321,6 +321,9 @@ export class TaskOrchestrator { batch: Batch, groupId: number ) { + const applyFromCacheOrRunBatchStart = performance.mark( + 'TaskOrchestrator-apply-from-cache-or-run-batch:start' + ); const taskEntries = Object.entries(batch.taskGraph.tasks); const tasks = taskEntries.map(([, task]) => task); @@ -374,9 +377,19 @@ export class TaskOrchestrator { groupId ); } + // Batch is done, mark it as completed + const applyFromCacheOrRunBatchEnd = performance.mark( + 'TaskOrchestrator-apply-from-cache-or-run-batch:end' + ); + performance.measure( + 'TaskOrchestrator-apply-from-cache-or-run-batch', + applyFromCacheOrRunBatchStart.name, + applyFromCacheOrRunBatchEnd.name + ); } private async runBatch(batch: Batch, env: NodeJS.ProcessEnv) { + const runBatchStart = performance.mark('TaskOrchestrator-run-batch:start'); try { const batchProcess = await this.forkedProcessTaskRunner.forkProcessForBatch( @@ -402,6 +415,13 @@ export class TaskOrchestrator { task: this.taskGraph.tasks[rootTaskId], status: 'failure' as TaskStatus, })); + } finally { + const runBatchEnd = performance.mark('TaskOrchestrator-run-batch:end'); + performance.measure( + 'TaskOrchestrator-run-batch', + runBatchStart.name, + runBatchEnd.name + ); } } diff --git a/settings.gradle.kts b/settings.gradle.kts index 0c548fd01a..6491576060 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -16,3 +16,4 @@ pluginManagement { rootProject.name = "nx" includeBuild("./packages/gradle/project-graph") +includeBuild("./packages/gradle/batch-runner")