Fix Windows build and add Windows CPU tests #145
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and test Windows wheel | ||
| on: | ||
| pull_request: | ||
| push: | ||
| branches: | ||
| - nightly | ||
| - main | ||
| - release/* | ||
| tags: | ||
| - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ | ||
| workflow_dispatch: | ||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }} | ||
| cancel-in-progress: true | ||
| permissions: | ||
| id-token: write | ||
| contents: write | ||
| defaults: | ||
| run: | ||
| shell: bash -l -eo pipefail {0} | ||
| jobs: | ||
| generate-matrix: | ||
| uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main | ||
| with: | ||
| package-type: wheel | ||
| os: windows | ||
| test-infra-repository: pytorch/test-infra | ||
| test-infra-ref: main | ||
| with-xpu: disable | ||
| with-rocm: disable | ||
| with-cuda: disable | ||
| build-python-only: "disable" | ||
| # Explicitly avoid 3.13 because 3.13t builds don't work. | ||
| # TODO remove eventually. | ||
| python-versions: ["3.9", "3.10", "3.11", "3.12"] | ||
| build: | ||
| needs: generate-matrix | ||
| strategy: | ||
| fail-fast: false | ||
| name: Build and Upload Windows wheel | ||
| # TODO: use @main | ||
| uses: nicolashug/test-infra/.github/workflows/build_wheels_windows.yml@build-platform-windows | ||
| with: | ||
| repository: pytorch/torchcodec | ||
| ref: "" | ||
| test-infra-repository: nicolashug/test-infra | ||
| test-infra-ref: build-platform-windows | ||
| build-matrix: ${{ needs.generate-matrix.outputs.matrix }} | ||
| pre-script: packaging/pre_build_script.sh | ||
| # post-script: packaging/post_build_script.sh TODO: consider enabling post-build checks for Windows | ||
| env-script: packaging/vc_env_helper.bat | ||
| smoke-test-script: packaging/fake_smoke_test.py | ||
| package-name: torchcodec | ||
| trigger-event: ${{ github.event_name }} | ||
| build-platform: "python-build-package" | ||
| # The BUILD_AGAINST_ALL_FFMPEG_FROM_S3 var, needed to build the wheel, is | ||
| # set in vc_env_helper.bat Couldn't find a way to set it from here. | ||
| build-command: "python -m build --wheel -vvv --no-isolation" | ||
| install-and-test: | ||
| runs-on: windows-latest | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| python-version: ['3.9', '3.10', '3.11', '3.12'] | ||
| # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] | ||
| ffmpeg-version-for-tests: ['6.1.1'] | ||
| needs: build | ||
| steps: | ||
| - uses: actions/download-artifact@v4 | ||
| with: | ||
| name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_x64 | ||
| path: pytorch/torchcodec/dist/ | ||
| - name: Setup conda env | ||
| uses: conda-incubator/setup-miniconda@v2 | ||
| with: | ||
| auto-update-conda: true | ||
| miniconda-version: "latest" | ||
| activate-environment: test | ||
| python-version: ${{ matrix.python-version }} | ||
| - name: Update pip | ||
| run: python -m pip install --upgrade pip | ||
| - name: Install PyTorch | ||
| run: | | ||
| python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu | ||
| - name: Install torchcodec from the wheel | ||
| run: | | ||
| wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"` | ||
| echo Installing $wheel_path | ||
| python -m pip install $wheel_path -vvv | ||
| - name: Check out repo | ||
| uses: actions/checkout@v3 | ||
| - name: Install ffmpeg, post build | ||
| run: | | ||
| # Ideally we would have checked for that before installing the wheel, | ||
| # but we need to checkout the repo to access this file, and we don't | ||
| # want to checkout the repo before installing the wheel to avoid any | ||
| # side-effect. It's OK. | ||
| source packaging/helpers.sh | ||
| assert_ffmpeg_not_installed | ||
| conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge | ||
| ffmpeg -version | ||
| # On Windows, ensure the conda Library/bin directory is in PATH | ||
| # This is needed for Windows DLL loading to find FFmpeg DLLs | ||
| if [[ "$RUNNER_OS" == "Windows" ]]; then | ||
| conda_env_path=$(conda info --base)/envs/test | ||
| library_bin_path="$conda_env_path/Library/bin" | ||
| echo "Adding conda Library/bin to PATH: $library_bin_path" | ||
| echo "$library_bin_path" >> $GITHUB_PATH | ||
| # Verify FFmpeg DLLs are accessible | ||
| echo "Checking if FFmpeg DLLs are in PATH:" | ||
| where avutil.dll || echo "avutil.dll not found in PATH" | ||
| where avcodec.dll || echo "avcodec.dll not found in PATH" | ||
| where avformat.dll || echo "avformat.dll not found in PATH" | ||
| fi | ||
| - name: Test torchcodec import after FFmpeg installation | ||
| run: | | ||
| echo "Testing torchcodec import after FFmpeg is installed and PATH is updated..." | ||
| python -c "import torchcodec; print('TorchCodec import successful!')" | ||
| - name: Install test dependencies | ||
| run: | | ||
| # Ideally we would find a way to get those dependencies from pyproject.toml | ||
| python -m pip install numpy pytest pillow | ||
| - name: Delete the src/ folder just for fun | ||
| run: | | ||
| # The only reason we checked-out the repo is to get access to the | ||
| # tests. We don't care about the rest. Out of precaution, we delete | ||
| # the src/ folder to be extra sure that we're running the code from | ||
| # the installed wheel rather than from the source. | ||
| # This is just to be extra cautious and very overkill because a) | ||
| # there's no way the `torchcodec` package from src/ can be found from | ||
| # the PythonPath: the main point of `src/` is precisely to protect | ||
| # against that and b) if we ever were to execute code from | ||
| # `src/torchcodec`, it would fail loudly because the built .so files | ||
| # aren't present there. | ||
| rm -r src/ | ||
| ls | ||
| - name: Run Python tests | ||
| run: | | ||
| pytest test -vvv | ||
| install-and-test-on-test-infra: | ||
| strategy: | ||
| matrix: | ||
| python-version: | ||
| - "3.9" | ||
| - "3.10" | ||
| - "3.11" | ||
| # - "3.12" | ||
| runner: ["windows.4xlarge"] | ||
| gpu-arch-type: ["cpu"] | ||
| ffmpeg-version-for-tests: ['6.1.1'] | ||
| # ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1'] | ||
| # TODO: put GPU testing back | ||
| # include: | ||
| # - python-version: "3.9" | ||
| # runner: windows.g5.4xlarge.nvidia.gpu | ||
| # gpu-arch-type: cuda | ||
| # gpu-arch-version: "11.8" | ||
| fail-fast: false | ||
| # needs: build | ||
| needs: generate-matrix ## TODO REMOVE | ||
| uses: pytorch/test-infra/.github/workflows/windows_job.yml@main | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| with: | ||
| repository: pytorch/torchcodec | ||
| runner: ${{ matrix.runner }} | ||
| gpu-arch-type: ${{ matrix.gpu-arch-type }} | ||
| gpu-arch-version: ${{ matrix.gpu-arch-version }} | ||
| timeout: 120 | ||
| test-infra-ref: main | ||
| script: | | ||
| set -euxo pipefail | ||
| export PYTHON_VERSION=${{ matrix.python-version }} | ||
| export VC_YEAR=2022 | ||
| export VSDEVCMD_ARGS="" | ||
| export GPU_ARCH_TYPE=${{ matrix.gpu-arch-type }} | ||
| export GPU_ARCH_VERSION=${{ matrix.gpu-arch-version }} | ||
| export FFMPEG_VERSION_FOR_TESTS=${{ matrix.ffmpeg-version-for-tests }} | ||
| export BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 | ||
| # Prepare conda | ||
| set +x && eval "$($(which conda) shell.bash hook)" && set -x | ||
| echo '::group::Create build environment' | ||
| conda create \ | ||
| --name ci \ | ||
| --quiet --yes \ | ||
| python="${PYTHON_VERSION}" pip | ||
| conda activate ci | ||
| # Install PyTorch | ||
| python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu | ||
| # Install build dependencies | ||
| conda install cmake pkg-config pybind11 "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge | ||
| # Ensure FFmpeg DLLs are in PATH for Windows | ||
| conda_env_path=$(conda info --base)/envs/ci | ||
| library_bin_path="$conda_env_path/Library/bin" | ||
| echo "Adding conda Library/bin to PATH: $library_bin_path" | ||
| echo "$library_bin_path" >> $GITHUB_PATH | ||
| # Verify FFmpeg is available | ||
| ffmpeg -version | ||
| # Install torchcodec from source in development mode | ||
| pip install -e ".[dev]" --no-build-isolation -vv | ||
| # Add torchcodec source directory to DLL search path for Windows | ||
| torchcodec_dll_path="$(pwd)/src/torchcodec" | ||
| echo "Adding torchcodec DLL path: $torchcodec_dll_path" | ||
| export PATH="$torchcodec_dll_path:$PATH" | ||
| # Test torchcodec import with detailed error reporting | ||
| python -c "import torch; import torchcodec" | ||
| # Run tests | ||
| pytest test -vvv | ||
