Merge branch 'main' into chore/update-changelog-prerelease

Author: cpsievert

.github/workflows/conventional-commits.yaml renamed to .github/workflows/verify-conventional-commits.yaml

@@ -0,0 +1,97 @@
+name: Verify test generation prompts
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/verify-test-generation-prompts.yml"
+      - "shiny/pytest/_generate/**"
+  workflow_dispatch:
+
+concurrency:
+  group: "prompt-test-generation-${{ github.event.pull_request.number || 'dispatch' }}"
+  cancel-in-progress: true
+
+env:
+  PYTHON_VERSION: "3.13"
+  ATTEMPTS: 3
+  PYTHONUNBUFFERED: 1
+
+jobs:
+  verify-test-generation-prompts:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Setup py-shiny
+        id: install
+        uses: ./.github/py-shiny/setup
+
+      - name: Install Test Generator Dependencies
+        run: |
+          make ci-install-ai-deps
+
+      - name: Run Evaluation and Tests 3 Times
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          PYTHONUNBUFFERED: 1
+        timeout-minutes: 25
+        run: |
+          make run-test-ai-evaluation
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results-${{ github.run_id }}
+          path: |
+            test-results-inspect-ai/
+          retention-days: 7
+
+      - name: Process Results
+        timeout-minutes: 2
+        run: |
+          # Results are already averaged by the bash script, just verify they exist
+          if [ ! -f "test-results-inspect-ai/summary.json" ]; then
+            echo "No averaged summary found at test-results-inspect-ai/summary.json"
+            ls -la test-results-inspect-ai/
+            exit 1
+          else
+            echo "Using averaged results from all attempts"
+            cat test-results-inspect-ai/summary.json
+          fi
+
+      - name: Check Quality Gate
+        timeout-minutes: 2
+        run: |
+          if [ ! -f "test-results-inspect-ai/summary.json" ]; then
+            echo "Summary file not found at test-results-inspect-ai/summary.json"
+            ls -la test-results-inspect-ai/
+            exit 1
+          else
+            echo "Found summary file, checking quality gate..."
+            python tests/inspect-ai/utils/scripts/quality_gate.py test-results-inspect-ai/
+          fi
+
+      - name: Prepare Comment Body
+        if: github.event_name == 'pull_request'
+        timeout-minutes: 1
+        run: |
+          python tests/inspect-ai/scripts/prepare_comment.py test-results-inspect-ai/summary.json
+
+      - name: Comment PR Results
+        if: github.event_name == 'pull_request'
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: inspect-ai-results
+          path: comment_body.txt

.github/workflows/verify-test-generation-prompts.yaml

@@ -0,0 +1,93 @@
+name: Verify testing documentation for changes
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/verify-testing-docs-on-change.yml"
+      - "docs/_quartodoc-testing.yml"
+      - "shiny/playwright/controller/**"
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  verify-testing-docs:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup py-shiny
+        id: install
+        uses: ./.github/py-shiny/setup
+
+      - name: Install dependencies
+        run: |
+          make ci-install-docs
+
+      - name: Update testing docs and check for changes
+        id: check-docs-changes
+        run: |
+          # Store the current state of the documentation file
+          cp shiny/pytest/_generate/_data/testing-documentation.json testing-documentation-before.json
+
+          # Run the make command to update testing docs
+          make update-testing-docs
+
+          if [[ ! -f testing-documentation-before.json || ! -f shiny/pytest/_generate/_data/testing-documentation.json ]]; then
+            echo "One or both documentation files are missing."
+            exit 1
+          fi
+
+          # Check if the documentation file has changed
+          if diff -q testing-documentation-before.json shiny/pytest/_generate/_data/testing-documentation.json > /dev/null 2>&1; then
+            echo "docs_changed=true" >> $GITHUB_OUTPUT
+            echo "The generated documentation is out of sync with the current controller changes."
+            echo "\n\n"
+            diff -q testing-documentation-before.json shiny/pytest/_generate/_data/testing-documentation.json || true
+            echo "\n\n"
+          else
+            echo "docs_changed=false" >> $GITHUB_OUTPUT
+            echo "Documentation file is up to date"
+          fi
+
+      - name: Comment on PR about testing docs update
+        if: steps.check-docs-changes.outputs.docs_changed == 'true'
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: testing-docs-update
+          message: |
+            🚨 **Testing Documentation Out of Sync**
+
+            We detected changes in the `shiny/playwright/controller` directory that affect the testing documentation used by the `shiny add test` command.
+
+            **The generated documentation is out of sync with your controller changes. Please run:**
+
+            ```bash
+            make update-testing-docs
+            ```
+
+            **Then commit the updated `shiny/pytest/_generate/_data/testing-documentation.json` file.**
+
+            <details><summary>Additional details</summary>
+
+            The updated documentation file ensures that the AI test generator has access to the latest controller API documentation.
+
+            </details>
+
+            ❌ **This check will fail until the documentation is updated and committed.**
+
+            ---
+            *This comment was automatically generated by the `verify-testing-docs-on-change.yml` workflow.*
+
+      - name: Remove comment when no controller changes or docs are up to date
+        if: steps.check-docs-changes.outputs.docs_changed == 'false'
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: testing-docs-update
+          delete: true

.github/workflows/verify-testing-docs-on-change.yml

@@ -123,3 +123,10 @@ shiny_bookmarks/
 
 # setuptools_scm
 shiny/_version.py
+
+# Other
+tests/inspect-ai/apps/*/test_*.py
+test-results.xml
+results-inspect-ai/
+test-results-inspect-ai/
+tests/inspect-ai/scripts/test_metadata.json

.gitignore

@@ -7,78 +7,73 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [1.5.0] - 2025-09-08
 
-### Breaking changes
-
-* The `ui.Chat` and `ui.MarkdownStream` components are now imported from the new `shinychat` library. Future versions of `shinychat` will likely deprecate and remove some features from `Chat`. If you still want to use those features with the latest Shiny, we suggest pinning `shinychat` to it's initial release (v0.1.0). (#2051)
-
 ### New features
 
-* Added `ui.insert_nav_panel()`, `ui.remove_nav_panel()`, and `ui.update_nav_panel()` to support dynamic navigation. (#90)
+* Added AI-powered test generator for Shiny applications. Use `shiny add test` to automatically generate comprehensive Playwright tests for your apps using AI models from Anthropic or OpenAI. (#2041)
 
 * `ui.sidebar()` is now interactively resizable. (#2020)
 
-* `navset_card_*()` now has a `full_screen` option to support `card()`'s existing full-screen functionality. (#1451)
+* `ui.sidebar()` gains a `fillable` argument to support vertical fill behavior in sidebars. (#2077)
 
-* `ui.update_*()` functions now accept `ui.TagChild` (i.e., HTML) as input to the `label` and `icon` arguments. (#2020)
+* Added `ui.insert_nav_panel()`, `ui.remove_nav_panel()`, and `ui.update_nav_panel()` to support dynamic navigation. (#90)
 
-* Added support for python 3.13. (#1711)
+* `navset_card_*()` now gains a `full_screen` option. (#1451)
+
+* `ui.update_*()` functions now accept `ui.TagChild` (i.e., HTML) as input to the `label` and `icon` arguments. (#2020)
 
 * The `.output_*()` methods of the `ClientData` class (e.g., `session.clientdata.output_height()`) can now be called without an `id` when called inside a `@render` function. (#1978)
 
 * `playwright.controller.InputActionButton` gains a `expect_icon()` method. As a result, the already existing `expect_label()` no longer includes the icon. (#2020)
 
-### Changes
+### Breaking changes
+
+* The `ui.Chat` and `ui.MarkdownStream` components are now imported from the new `shinychat` library. Future versions of `shinychat` will likely deprecate and remove some features from `Chat`. If you still want to use those features with the latest Shiny, we suggest pinning `shinychat` to it's initial release (v0.1.0). (#2051)
 
 * `express.ui.insert_accordion_panel()`'s function signature has changed to be more ergonomic. Now you can pass the `panel_title` and `panel_contents` directly instead of `ui.hold()`ing the `ui.accordion_panel()` context manager. (#2042)
 
 ### Improvements
 
-* `input_date()`, `input_date_range()`, `update_date()`, and `update_date_range()` now supports `""` for values, mins, and maxes. In this case, no date will be specified on the client. (#1713) (#1689)
-
-* Restricted the allowable types of the `choices` parameter of `input_select()`, `input_selectize()`, `update_select()`, and `update_selectize()` to actual set of allowable types (previously, the type was suggesting HTML-like values were supported). (#2048)
-
 * Improved the styling and readability of markdown tables rendered by `ui.Chat()` and `ui.MarkdownStream()`. (#1973)
 
-* `selectize`, `remove_button`, and `options` parameters of `ui.input_select()` have been deprecated; use `ui.input_selectize()` instead. (Thanks, @ErdaradunGaztea!) (#1947)
+* `input_date()`, `input_date_range()`, `update_date()`, and `update_date_range()` now support `""` for values, mins, and maxes. In this case, no date will be specified on the client. (#1713) (#1689)
 
-* Added `timeout_secs` parameter to `create_app_fixture` to allow testing apps with longer startup times. (#2033)
+* Restricted the allowable types of the `choices` parameter of `input_select()`, `input_selectize()`, `update_select()`, and `update_selectize()` to actual set of allowable types (previously, the type was suggesting HTML-like values were supported). (#2048)
 
 * Added module support for `session.clientdata` methods. This allows you to access client data values in Shiny modules without needing to namespace the keys explicitly. (#1978)
 
-* Add support for selecting menu items in `Navset` controllers to improve dropdown navigation test coverage. (#2066)
+* Added `timeout_secs` parameter to `create_app_fixture` to allow testing apps with longer startup times. (#2033)
 
-* Fixed false positive warning in `layout_columns()` about number of widths vs elements. (#1704)
+* Add support for selecting menu items in `Navset` controllers to improve dropdown navigation test coverage. (#2066)
 
-* When errors occur in a bookmarking context, they are now reported in the Python console. (#2076)
+* Python 3.13 is now offically supported and tested. (#1711)
 
 ### Bug fixes
 
+* `include_js()` and `include_css()` now work as expected in multi-user settings and also when multiple files from the same directory are included. (#2061, #2069)
+
 * Fixed numerous issues related to programmatically updating selectize options. (#2053)
    * `update_selectize(options=...)` no longer gets ignored when `server=False` (the default).
    * `update_selectize(options=...)` now works as expected in a module.
 
-* Fixed an issue with `update_selectize()` to properly display labels with HTML reserved characters like "&" (#1330)
+* Fixed an issue with `update_selectize(server=True)` not properly displaying labels with HTML reserved characters like "&" (#1330)
 
 * Fixed an issue with `ui.Chat()` sometimes wanting to scroll a parent element. (#1996)
 
-* Explicitly call out module usage in UI input bookmark button documentation. (#1983)
-
-* Fix missing session when trying to display an error duing bookmarking. (#1984)
+* Fix several issues with bookmarking error reporting and documentation. (#2076, #1984, #1983)
 
 * `input_date()` and `input_date_range()` once again use the client's (not the server) current date as the default `value`. (#2060)
 
-* Fixed `set()` method of `InputSelectize` controller so it clears existing selections before applying new values. (#2024)
-
-* `include_js()` and `include_css()` now work as expected when trying to include multiple files from the same directory. (#2069)
+* Fixed false positive warning in `layout_columns()` about number of widths vs elements. (#1704)
 
-* `include_js()` and `include_css()` now correctly handle file permissions in multi-user settings. (#2061)
+* Fixed `set()` method of the `InputSelectize` test controller so it clears existing selections before applying new values. (#2024)
 
 ### Deprecations
 
-* `ui.update_navs()` has been deprecated in favor of `ui.update_navset()`. (#2047)
+* `ui.update_navs()` is deprecated in favor of `ui.update_navset()`. (#2047)
 
 * `ui.panel_well()` is deprecated in favor of `ui.card()`. (#2038)
 
+* `selectize`, `remove_button`, and `options` parameters of `ui.input_select()` have been deprecated; use `ui.input_selectize()` instead. (Thanks, @ErdaradunGaztea!) (#1947)
 
 ## [1.4.0] - 2025-04-08
 

CHANGELOG.md

@@ -123,6 +123,35 @@ docs-quartodoc: FORCE
 	@echo "-------- Making quartodoc docs --------"
 	@cd docs && make quartodoc
 
+install-repomix: install-npm FORCE ## Install repomix if not already installed
+	@echo "-------- Installing repomix if needed --------"
+	@if ! command -v repomix > /dev/null 2>&1; then \
+		echo "Installing repomix..."; \
+		npm install -g repomix; \
+	else \
+		echo "repomix is already installed"; \
+	fi
+
+update-testing-docs-repomix: install-repomix FORCE ## Generate repomix output for testing docs
+	@echo "-------- Generating repomix output for testing docs --------"
+	repomix docs/api/testing -o tests/inspect-ai/utils/scripts/repomix-output-testing.xml
+
+update-testing-docs-process: FORCE ## Process repomix output to generate testing documentation JSON
+	@echo "-------- Processing testing documentation --------"
+	python tests/inspect-ai/utils/scripts/process_docs.py --input tests/inspect-ai/utils/scripts/repomix-output-testing.xml --output shiny/pytest/_generate/_data/testing-documentation.json
+	@echo "-------- Cleaning up temporary files --------"
+	rm -f tests/inspect-ai/utils/scripts/repomix-output-testing.xml
+
+update-testing-docs: docs update-testing-docs-repomix update-testing-docs-process FORCE ## Update testing documentation (full pipeline)
+	@echo "-------- Testing documentation update complete --------"
+
+ci-install-ai-deps: FORCE
+	uv pip install -e ".[dev,test,testgen]"
+	$(MAKE) install-playwright
+
+run-test-ai-evaluation: FORCE ## Run the AI evaluation script for tests
+	@echo "-------- Running AI evaluation for tests --------"
+	bash ./tests/inspect-ai/scripts/run-test-evaluation.sh
 
 install-npm: FORCE
 	$(if $(shell which npm), @echo -n, $(error Please install node.js and npm first. See https://nodejs.org/en/download/ for instructions.))

Makefile

@@ -124,6 +124,13 @@ doc = [
     "quartodoc>=0.8.1",
     "griffe>=1.3.2",
 ]
+testgen = [
+    "chatlas[anthropic,openai]",
+    "openai>=1.104.1",
+    "anthropic>=0.62.0",
+    "inspect-ai>=0.3.129",
+    "pytest-timeout",
+]
 
 
 [project.urls]

pyproject.toml

@@ -10,7 +10,10 @@
     "docs",
     "tests/playwright/deploys/*/app.py",
     "shiny/templates",
-    "tests/playwright/ai_generated_apps",
+    "tests/playwright/ai_generated_apps/*/*/app*.py",
+    "tests/inspect-ai/apps/*/app*.py",
+    "shiny/pytest/_generate/_main.py",
+    "tests/inspect-ai/scripts/evaluation.py"
   ],
   "typeCheckingMode": "strict",
   "reportImportCycles": "none",