Skip to content

Commit 9479e4b

Browse files
committed
test(integration): add NanoDSL LLM integration test suite
Introduce integration tests for NanoDSL generation via LLM, verifying DSL compilation for various user prompts. Update build config to support integrationTest source set and task.
1 parent 4f52356 commit 9479e4b

19 files changed

+1589
-10
lines changed

.github/workflows/build.yml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ jobs:
8888
id: artifact
8989
shell: bash
9090
run: |
91-
cd ${{ github.workspace }}/build/distributions
91+
cd ${{ github.workspace }}/mpp-idea/build/distributions
9292
FILENAME=`ls *.zip`
9393
unzip "$FILENAME" -d content
9494
@@ -99,7 +99,7 @@ jobs:
9999
uses: actions/upload-artifact@v4
100100
with:
101101
name: ${{ steps.artifact.outputs.filename }}
102-
path: ./build/distributions/content/*/*
102+
path: ./mpp-idea/build/distributions/content/*/*
103103

104104
# Run tests and upload a code coverage report
105105
test:
@@ -134,21 +134,21 @@ jobs:
134134

135135
# Run tests
136136
- name: Run Tests
137-
run: ./gradlew check
137+
run: ./gradlew :mpp-idea:check
138138

139139
# Collect Tests Result of failed tests
140140
- name: Collect Tests Result
141141
if: ${{ failure() }}
142142
uses: actions/upload-artifact@v4
143143
with:
144144
name: tests-result
145-
path: ${{ github.workspace }}/build/reports/tests
145+
path: ${{ github.workspace }}/mpp-idea/build/reports/tests
146146

147147
# Upload the Kover report to CodeCov
148148
- name: Upload Code Coverage Report
149149
uses: codecov/codecov-action@v4
150150
with:
151-
files: ${{ github.workspace }}/build/reports/kover/report.xml
151+
files: ${{ github.workspace }}/mpp-idea/build/reports/kover/report.xml
152152

153153
# Run plugin structure verification along with IntelliJ Plugin Verifier
154154
verify:
@@ -198,7 +198,7 @@ jobs:
198198
uses: actions/upload-artifact@v4
199199
with:
200200
name: pluginVerifier-result
201-
path: ${{ github.workspace }}/build/reports/pluginVerifier
201+
path: ${{ github.workspace }}/mpp-idea/build/reports/pluginVerifier
202202

203203
# Prepare a draft release for GitHub Releases page for the manual verification
204204
# If accepted and published, release workflow would be triggered

xuiper-ui/build.gradle.kts

Lines changed: 49 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,22 +11,44 @@ repositories {
1111
mavenCentral()
1212
}
1313

14+
// Configure integration test source set
15+
sourceSets {
16+
create("integrationTest") {
17+
kotlin.srcDir("src/integrationTest/kotlin")
18+
resources.srcDir("src/integrationTest/resources")
19+
compileClasspath += sourceSets.main.get().output + sourceSets.test.get().output
20+
runtimeClasspath += sourceSets.main.get().output + sourceSets.test.get().output
21+
}
22+
}
23+
24+
val integrationTestImplementation by configurations.getting {
25+
extendsFrom(configurations.testImplementation.get())
26+
}
27+
28+
val integrationTestRuntimeOnly by configurations.getting {
29+
extendsFrom(configurations.testRuntimeOnly.get())
30+
}
31+
1432
dependencies {
1533
implementation(project(":mpp-core"))
1634

1735
// Serialization for test case format
1836
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.8.0")
19-
37+
2038
// Coroutines
2139
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.0")
22-
40+
2341
// Logging
2442
implementation("io.github.oshai:kotlin-logging-jvm:7.0.0")
2543
implementation("ch.qos.logback:logback-classic:1.5.6")
26-
44+
2745
// Testing
2846
testImplementation(kotlin("test"))
2947
testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.8.0")
48+
49+
// Integration test dependencies
50+
integrationTestImplementation(kotlin("test"))
51+
integrationTestImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.8.0")
3052
}
3153

3254
kotlin {
@@ -37,13 +59,36 @@ tasks.test {
3759
useJUnitPlatform()
3860
}
3961

62+
// Integration test task
63+
tasks.register<Test>("integrationTest") {
64+
group = "verification"
65+
description = "Run NanoDSL integration tests that call LLM and verify DSL compilation"
66+
testClassesDirs = sourceSets["integrationTest"].output.classesDirs
67+
classpath = sourceSets["integrationTest"].runtimeClasspath
68+
useJUnitPlatform()
69+
70+
// Pass environment variables for LLM configuration
71+
environment("OPENAI_API_KEY", System.getenv("OPENAI_API_KEY") ?: "")
72+
environment("ANTHROPIC_API_KEY", System.getenv("ANTHROPIC_API_KEY") ?: "")
73+
environment("DEEPSEEK_API_KEY", System.getenv("DEEPSEEK_API_KEY") ?: "")
74+
75+
// Integration tests may take longer
76+
systemProperty("junit.jupiter.execution.timeout.default", "5m")
77+
78+
// Show output for debugging
79+
testLogging {
80+
events("passed", "skipped", "failed")
81+
showStandardStreams = true
82+
}
83+
}
84+
4085
// Custom task to run DSL evaluation tests
4186
tasks.register<JavaExec>("runDslEval") {
4287
group = "verification"
4388
description = "Run NanoDSL AI evaluation tests"
4489
mainClass.set("cc.unitmesh.xuiper.eval.DslEvalRunnerKt")
4590
classpath = sourceSets["main"].runtimeClasspath
46-
91+
4792
// Pass environment variables for LLM configuration
4893
environment("OPENAI_API_KEY", System.getenv("OPENAI_API_KEY") ?: "")
4994
environment("ANTHROPIC_API_KEY", System.getenv("ANTHROPIC_API_KEY") ?: "")

0 commit comments

Comments
 (0)