Add cuml, raft-dask

.pre-commit-config.yaml

@@ -25,13 +25,12 @@ repos:
             .*test.*|
             ^CHANGELOG.md$
           )
-  # Include this if we add any Python code
-  # - repo: https://github.com/astral-sh/ruff-pre-commit
-  #   rev: v0.9.3
-  #   hooks:
-  #     - id: ruff
-  #       args: ["--fix"]
-  #     - id: ruff-format
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.3
+    hooks:
+      - id: ruff
+        args: ["--fix"]
+      - id: ruff-format
   - repo: https://github.com/rapidsai/pre-commit-hooks
     rev: v0.6.0
     hooks:

README.md

@@ -2,7 +2,8 @@
 
 [![Test dask-upstream](https://github.com/rapidsai/dask-upstream-testing/actions/workflows/cron.yaml/badge.svg)](https://github.com/rapidsai/dask-upstream-testing/actions/workflows/cron.yaml)
 
-This repository contains the scripts to run Dask's `gpu`-marked tests on a schedule.
+This repository contains the scripts to run Dask's `gpu`-marked tests on a schedule
+and dask-dependent tests from some downstream libraries.
 
 ## Version Policy
 

requirements/overrides.txt

@@ -0,0 +1,2 @@
+dask @ git+https://github.com/dask/dask.git@main
+distributed @ git+https://github.com/dask/distributed.git@main

scripts/check-version.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# SPDX-FileCopyrightText: Copyright (c) 2023-2025, NVIDIA CORPORATION & AFFILIATES.
+"""
+Print the git commit a rapids package was built from.
+"""
+
+import argparse
+import importlib
+import sys
+import importlib.resources
+
+
+def parse_args(args=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("distribution", help="Package name to check.")
+
+    return parser.parse_args(args)
+
+
+def main(args=None):
+    args = parse_args(args)
+    dist = args.distribution
+
+    try:
+        sha = importlib.resources.files(dist).joinpath("GIT_COMMIT").read_text().strip()
+    except ModuleNotFoundError:
+        print(f"Error: {dist} is not installed.", file=sys.stderr)
+    except FileNotFoundError:
+        print(f"Error: {dist} does not contain 'GIT_COMMIT' file.", file=sys.stderr)
+    else:
+        print(sha)
+        sys.exit(0)
+    sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

scripts/install.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES.
+set -euo pipefail
+
+# RAPIDS_CUDA_VERSION is like 12.15.1
+# We want cu12
+RAPIDS_PY_CUDA_SUFFIX=$(echo "cu${RAPIDS_CUDA_VERSION:-12.15.1}" | cut -d '.' -f 1)
+
+DASK_VERSION=main
+
+# Try
+uv pip install --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple \
+  --overrides=requirements/overrides.txt \
+  --prerelease allow \
+  "cudf-${RAPIDS_PY_CUDA_SUFFIX}" \
+  "dask-cudf-${RAPIDS_PY_CUDA_SUFFIX}" \
+  "ucx-py-${RAPIDS_PY_CUDA_SUFFIX}" \
+  "ucxx-${RAPIDS_PY_CUDA_SUFFIX}" \
+  "scipy" \
+  "dask-cuda"
+
+# Clone cudf repo for tests
+CUDF_VERSION="branch-25.04"
+cudf_commit=$(./scripts/check-version.py cudf)
+
+if [ ! -d "cudf" ]; then
+    echo "Cloning cudf@{$CUDF_VERSION}"
+    git clone https://github.com/rapidsai/cudf.git --branch $CUDF_VERSION
+fi
+
+pushd cudf
+git checkout $cudf_commit
+popd
+
+if [ ! -d "dask-cuda" ]; then
+    echo "Cloning cudf@{$CUDF_VERSION}"
+    git clone https://github.com/rapidsaicudf_commit/dask-cuda.git --branch $CUDF_VERSION
+fi
+
+# Clone dask-cuda for tests
+# dask-cuda nightly wheels currently lack a __git_commit__.
+# Looking into it, but for now just use the branch.
+
+# dask_cuda_commit=$(./scripts/check-version.py dask_cuda)
+
+pushd dask-cuda
+git checkout $CUDF_VERSION
+popd
+
+# depth needs to be sufficient to reach the last tag, so that the package
+# versions are set correctly
+if [ ! -d "dask" ]; then
+    echo "Cloning dask@{$DASK_VERSION}"
+    git clone https://github.com/dask/dask --depth 100 --branch $DASK_VERSION
+fi
+
+if [ ! -d "distributed" ]; then
+    echo "Cloning dask@{$DASK_VERSION}"
+    git clone https://github.com/dask/distributed --depth 100 --branch $DASK_VERSION
+fi
+
+pushd dask
+git checkout $DASK_VERSION
+popd
+
+pushd distributed
+git checkout $DASK_VERSION
+popd
+
+echo "[Setup done]"
+uv pip list

scripts/overrides.txt

@@ -0,0 +1,4 @@
+# used to force installing dask / distributed main
+# even if another package like rapids-dask-dependency wants something else
+dask[test] @ git+https://github.com/dask/dask.git@main
+distributed @ git+https://github.com/dask/distributed.git@main

scripts/run.sh

@@ -1,63 +1,8 @@
 #!/usr/bin/env bash
 # SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES.
 
-# Install
 set -euo pipefail
 
-
-# RAPIDS_CUDA_VERSION is like 12.15.1
-# We want cu12
-RAPIDS_PY_CUDA_SUFFIX=$(echo "cu${RAPIDS_CUDA_VERSION:-12.15.1}" | cut -d '.' -f 1)
-
-# TODO: set this to main once dask-cudf is compatible
-# DASK_VERSION=main
-DASK_VERSION=main
-export PIP_YES=true
-export PIP_PRE=true
-
-pip install --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple \
-  "cudf-${RAPIDS_PY_CUDA_SUFFIX}" \
-  "dask-cudf-${RAPIDS_PY_CUDA_SUFFIX}" \
-  "ucx-py-${RAPIDS_PY_CUDA_SUFFIX}" \
-  "ucxx-${RAPIDS_PY_CUDA_SUFFIX}" \
-  "scipy" \
-  "dask-cuda"
-
-
-# Clone cudf repo for tests
-CUDF_VERSION="branch-25.04"
-
-if [ ! -d "cudf" ]; then
-    echo "Cloning cudf@{$CUDF_VERSION}"
-    git clone https://github.com/rapidsai/cudf.git --branch $CUDF_VERSION
-fi
-
-# Clone dask-cuda for tests
-
-if [ ! -d "dask-cuda" ]; then
-    echo "Cloning cudf@{$CUDF_VERSION}"
-    git clone https://github.com/rapidsai/dask-cuda.git --branch $CUDF_VERSION
-fi
-
-
-# depth needs to be sufficient to reach the last tag, so that the package
-# versions are set correctly
-if [ ! -d "dask" ]; then
-    echo "Cloning dask@{$DASK_VERSION}"
-    git clone https://github.com/dask/dask --depth 100 --branch $DASK_VERSION
-fi
-
-if [ ! -d "distributed" ]; then
-    echo "Cloning dask@{$DASK_VERSION}"
-    git clone https://github.com/dask/distributed --depth 100 --branch $DASK_VERSION
-fi
-
-# Install everything, including any new dependencies
-pip uninstall dask distributed
-pip install -e ./dask[test]
-pip install -e ./distributed
-
-echo "[Setup done]"
-pip list
-
+./scripts/setup.sh
+./scripts/install.sh
 ./scripts/test.sh

scripts/setup.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES.
+
+# Install
+set -euo pipefail
+
+if ! command -v uv > /dev/null; then
+    curl -LsSf https://astral.sh/uv/install.sh | sh
+fi