diff --git a/.github/workflows/cron.yaml b/.github/workflows/cron.yaml index 6f6a508..4768126 100644 --- a/.github/workflows/cron.yaml +++ b/.github/workflows/cron.yaml @@ -3,11 +3,11 @@ name: Test dask-upstream on: schedule: - # 18:15 UTC daily. - # We want to run after the nightly pipeline finishes. + # 06:15 and 08:15 UTC daily. + # We want at least one one run after the nightly pipeline finishes. # https://github.com/rapidsai/workflows/blob/main/.github/workflows/nightly-pipeline-trigger.yaml is # currently set to 5:00 UTC and takes ~12 hours - - cron: "15 18 * * *" + - cron: "15 06,18 * * *" workflow_dispatch: {} jobs: diff --git a/.gitignore b/.gitignore index 9cbc3b0..e82ebba 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ dask distributed cudf dask-cuda +packages diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7833564..f1f0948 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,13 +25,12 @@ repos: .*test.*| ^CHANGELOG.md$ ) - # Include this if we add any Python code - # - repo: https://github.com/astral-sh/ruff-pre-commit - # rev: v0.9.3 - # hooks: - # - id: ruff - # args: ["--fix"] - # - id: ruff-format + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.9.3 + hooks: + - id: ruff + args: ["--fix"] + - id: ruff-format - repo: https://github.com/rapidsai/pre-commit-hooks rev: v0.6.0 hooks: diff --git a/README.md b/README.md index dc08e62..b8cb20c 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ [![Test dask-upstream](https://github.com/rapidsai/dask-upstream-testing/actions/workflows/cron.yaml/badge.svg)](https://github.com/rapidsai/dask-upstream-testing/actions/workflows/cron.yaml) -This repository contains the scripts to run Dask's `gpu`-marked tests on a schedule. +This repository contains the scripts to run Dask's `gpu`-marked tests on a schedule +and dask-dependent tests from some downstream libraries. ## Version Policy diff --git a/requirements/overrides.txt b/requirements/overrides.txt new file mode 100644 index 0000000..3e3ebc7 --- /dev/null +++ b/requirements/overrides.txt @@ -0,0 +1,2 @@ +dask @ git+https://github.com/dask/dask.git@main +distributed @ git+https://github.com/dask/distributed.git@main diff --git a/scripts/check-version.py b/scripts/check-version.py new file mode 100755 index 0000000..8bbd0a0 --- /dev/null +++ b/scripts/check-version.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# SPDX-FileCopyrightText: Copyright (c) 2023-2025, NVIDIA CORPORATION & AFFILIATES. +""" +Print the git commit a rapids package was built from. +""" + +import argparse +import importlib +import sys +import importlib.resources + + +def parse_args(args=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("distribution", help="Package name to check.") + + return parser.parse_args(args) + + +def main(args=None): + args = parse_args(args) + dist = args.distribution + + try: + sha = importlib.resources.files(dist).joinpath("GIT_COMMIT").read_text().strip() + except ModuleNotFoundError: + print(f"Error: {dist} is not installed.", file=sys.stderr) + except FileNotFoundError: + print(f"Error: {dist} does not contain 'GIT_COMMIT' file.", file=sys.stderr) + else: + print(sha) + sys.exit(0) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 0000000..e5c74b2 --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. +set -euo pipefail + +# RAPIDS_CUDA_VERSION is like 12.15.1 +# We want cu12 +RAPIDS_PY_CUDA_SUFFIX=$(echo "cu${RAPIDS_CUDA_VERSION:-12.15.1}" | cut -d '.' -f 1) + +uv pip install --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple \ + --overrides=requirements/overrides.txt \ + --prerelease allow \ + "cuml-${RAPIDS_PY_CUDA_SUFFIX}[test]" \ + "cudf-${RAPIDS_PY_CUDA_SUFFIX}" \ + "dask-cudf-${RAPIDS_PY_CUDA_SUFFIX}" \ + "raft-dask-${RAPIDS_PY_CUDA_SUFFIX}" \ + "ucx-py-${RAPIDS_PY_CUDA_SUFFIX}" \ + "ucxx-${RAPIDS_PY_CUDA_SUFFIX}" \ + "scipy" \ + "dask-cuda" + +# packages holds all the downstream and upstream dependencies. +# we want to avoid directories with the same name as packages +# in the working directory +mkdir -p packages + +# Clone cudf repo for tests +CUDF_VERSION="branch-25.04" + +cudf_commit=$(./scripts/check-version.py cudf) + +if [ ! -d "cudf" ]; then + echo "Cloning cudf@{$CUDF_VERSION}" + git clone https://github.com/rapidsai/cudf.git --branch $CUDF_VERSION packages +fi + +pushd packages/cudf +git checkout $cudf_commit +popd + +cuml_commit=$(./scripts/check-version.py cuml) + +if [ ! -d "cuml" ]; then + echo "Cloning cuml@{$CUDF_VERSION}" + git clone https://github.com/rapidsai/cuml.git --branch $CUDF_VERSION packages +fi + +pushd packages/cuml +git checkout $cuml_commit +popd + +raft_commit=$(./scripts/check-version.py raft_dask) + +if [ ! -d "raft" ]; then + echo "Cloning raft@{$CUDF_VERSION}" + git clone https://github.com/rapidsai/raft.git --branch $CUDF_VERSION packages +fi + +pushd packages/raft +git checkout $raft_commit +popd + +if [ ! -d "dask-cuda" ]; then + echo "Cloning cudf@{$CUDF_VERSION}" + git clone https://github.com/rapidsaicudf_commit/dask-cuda.git --branch $CUDF_VERSION packages +fi + +# Clone dask-cuda for tests +# dask-cuda nightly wheels currently lack a __git_commit__. +# Looking into it, but for now just use the branch. + +# dask_cuda_commit=$(./scripts/check-version.py dask_cuda) + +pushd packages/dask-cuda +git checkout $CUDF_VERSION +popd + +# depth needs to be sufficient to reach the last tag, so that the package +# versions are set correctly +if [ ! -d "dask" ]; then + echo "Cloning dask@main" + git clone https://github.com/dask/dask --depth 100 packages +fi + +if [ ! -d "distributed" ]; then + echo "Cloning distributed@main" + git clone https://github.com/dask/distributed --depth 100 packages +fi + +pushd packages/dask +git checkout main +popd + +pushd packages/distributed +git checkout main +popd + +# Finally, ensure that +uv pip install --no-deps -e ./packages/dask ./packages/distributed + +echo "[Setup done]" +uv pip list diff --git a/scripts/overrides.txt b/scripts/overrides.txt new file mode 100644 index 0000000..1e35b4c --- /dev/null +++ b/scripts/overrides.txt @@ -0,0 +1,4 @@ +# used to force installing dask / distributed main +# even if another package like rapids-dask-dependency wants something else +dask[test] @ git+https://github.com/dask/dask.git@main +distributed @ git+https://github.com/dask/distributed.git@main diff --git a/scripts/run.sh b/scripts/run.sh index 872bf23..31e1b01 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -1,63 +1,8 @@ #!/usr/bin/env bash # SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. -# Install set -euo pipefail - -# RAPIDS_CUDA_VERSION is like 12.15.1 -# We want cu12 -RAPIDS_PY_CUDA_SUFFIX=$(echo "cu${RAPIDS_CUDA_VERSION:-12.15.1}" | cut -d '.' -f 1) - -# TODO: set this to main once dask-cudf is compatible -# DASK_VERSION=main -DASK_VERSION=main -export PIP_YES=true -export PIP_PRE=true - -pip install --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple \ - "cudf-${RAPIDS_PY_CUDA_SUFFIX}" \ - "dask-cudf-${RAPIDS_PY_CUDA_SUFFIX}" \ - "ucx-py-${RAPIDS_PY_CUDA_SUFFIX}" \ - "ucxx-${RAPIDS_PY_CUDA_SUFFIX}" \ - "scipy" \ - "dask-cuda" - - -# Clone cudf repo for tests -CUDF_VERSION="branch-25.04" - -if [ ! -d "cudf" ]; then - echo "Cloning cudf@{$CUDF_VERSION}" - git clone https://github.com/rapidsai/cudf.git --branch $CUDF_VERSION -fi - -# Clone dask-cuda for tests - -if [ ! -d "dask-cuda" ]; then - echo "Cloning cudf@{$CUDF_VERSION}" - git clone https://github.com/rapidsai/dask-cuda.git --branch $CUDF_VERSION -fi - - -# depth needs to be sufficient to reach the last tag, so that the package -# versions are set correctly -if [ ! -d "dask" ]; then - echo "Cloning dask@{$DASK_VERSION}" - git clone https://github.com/dask/dask --depth 100 --branch $DASK_VERSION -fi - -if [ ! -d "distributed" ]; then - echo "Cloning dask@{$DASK_VERSION}" - git clone https://github.com/dask/distributed --depth 100 --branch $DASK_VERSION -fi - -# Install everything, including any new dependencies -pip uninstall dask distributed -pip install -e ./dask[test] -pip install -e ./distributed - -echo "[Setup done]" -pip list - +./scripts/setup.sh +./scripts/install.sh ./scripts/test.sh diff --git a/scripts/setup.sh b/scripts/setup.sh new file mode 100755 index 0000000..fa249f2 --- /dev/null +++ b/scripts/setup.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. + +# Install +set -euo pipefail + +if ! command -v uv > /dev/null; then + curl -LsSf https://astral.sh/uv/install.sh | sh +fi diff --git a/scripts/test.sh b/scripts/test.sh index 5188595..b83b9ea 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -2,20 +2,27 @@ # SPDX-FileCopyrightText: Copyright (c) 2023-2025, NVIDIA CORPORATION & AFFILIATES. if [ $# -eq 0 ]; then + run_cuml=true run_dask=true run_dask_cuda=true run_dask_cudf=true run_distributed=true + run_raft_dask=true else + run_cuml=false run_dask=false run_dask_cuda=false run_dask_cudf=false run_distributed=false + run_raft_dask=false fi # Parse command-line arguments while [[ $# -gt 0 ]]; do case "$1" in + --cuml-only) + run_cuml=true + ;; --dask-only) run_dask=true ;; @@ -28,6 +35,9 @@ while [[ $# -gt 0 ]]; do --distributed-only) run_distributed=true ;; + --raft-dask-only) + run_raft_dask=true + ;; --help) echo "Usage: $0 [--dask-only] [--distributed-only] [--dask-cuda-only] [--dask-cudf-only]" exit 0 @@ -42,33 +52,41 @@ done exit_code=0; +# --- cuml --- +if $run_cuml; then + + echo "[testing cuml]" + pytest -v --quick_run packages/cuml/python/cuml/cuml/tests/dask + + if [[ $? -ne 0 ]]; then + exit_code=1 + fi + +fi + + # --- dask-cudf --- if $run_dask_cudf; then echo "[testing dask-cudf]" - pushd cudf/python/dask_cudf || exit 1 - pytest -v dask_cudf + pytest -v packages/cudf/python/dask_cudf if [[ $? -ne 0 ]]; then exit_code=1 fi - popd || exit 1 - fi # --- dask-cuda --- if $run_dask_cuda; then echo "[testing dask-cuda]" - pushd dask-cuda/dask_cuda/tests || exit 1 - pytest -v . + pytest -v packages/dask-cuda/dask_cuda/tests if [[ $? -ne 0 ]]; then exit_code=1 fi - popd || exit 1 fi # --- dask --- @@ -76,31 +94,39 @@ fi if $run_dask; then echo "[testing dask]" - pushd dask || exit 1 - pytest -v -m gpu dask + pytest -v -m gpu packages/dask/dask if [[ $? -ne 0 ]]; then exit_code=1 fi - popd || exit 1 +fi + +# --- raft-dask --- +if $run_raft_dask; then + + echo "[testing raft-dask]" + pytest -v packages/raft/python/raft-dask/raft_dask/tests + + if [[ $? -ne 0 ]]; then + exit_code=1 + fi fi -# --- dask --- + + +# --- distributed --- if $run_distributed; then echo "[testing distributed]" - pushd distributed || exit 1 - pytest -v -m gpu --runslow distributed + pytest -v -m gpu --runslow packages/distributed/distributed if [[ $? -ne 0 ]]; then exit_code=1 fi - popd || exit 1 - fi exit $exit_code