Build Mac wheels in CI

Author: scotts

.github/workflows/linux_wheel.yaml

@@ -38,7 +38,7 @@ jobs:
     needs: generate-matrix
     strategy:
       fail-fast: false
-    name: Build and Upload wheel
+    name: Build and Upload Linux wheel
     uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@main
     with:
       repository: pytorch/torchcodec

.github/workflows/macos_wheel.yaml

@@ -1,4 +1,4 @@
-name: Build and test MacOS
+name: Build and test MacOS wheels
 
 on:
   pull_request:
@@ -24,6 +24,35 @@ defaults:
     shell: bash -l -eo pipefail {0}
 
 jobs:
+  generate-matrix:
+    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
+    with:
+      package-type: wheel
+      os: macos-arm64
+      test-infra-repository: pytorch/test-infra
+      test-infra-ref: main
+      with-xpu: disable
+      with-rocm: disable
+      with-cuda: disable
+  build:
+    needs: generate-matrix
+    strategy:
+      fail-fast: false
+    name: Build and Upload Mac wheel
+    uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main
+    with:
+      repository: pytorch/torchcodec
+      ref: ""
+      test-infra-repository: pytorch/test-infra
+      test-infra-ref: main
+      build-matrix: ${{ needs.generate-matrix.outputs.matrix }}
+      post-script: packaging/post_build_script.sh
+      smoke-test-script: packaging/fake_smoke_test.py
+      runner-type: macos-m1-stable
+      package-name: torchcodec
+      trigger-event: ${{ github.event_name }}
+      env-var-script: packaging/wheel_env.sh
+
   install-and-test:
     runs-on: macos-m1-stable
     strategy:
@@ -32,7 +61,12 @@ jobs:
         python-version: ['3.9']
         ffmpeg-version-for-tests: ['4.4.2', '5.1.2', '6.1.1', '7.0.1']
     if: ${{ always() }}
+    needs: build
     steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: pytorch_torchcodec__${{ matrix.python-version }}_cpu_
+          path: pytorch/torchcodec/dist/
       - name: Setup conda env
         uses: conda-incubator/setup-miniconda@v3
         with:
@@ -45,19 +79,14 @@ jobs:
       - name: Install PyTorch
         run: |
           python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu
+      - name: Install torchcodec from the wheel
+        run: |
+          wheel_path=`find pytorch/torchcodec/dist -type f -name "*.whl"`
+          echo Installing $wheel_path
+          python -m pip install $wheel_path -vvv
+
       - name: Check out repo
         uses: actions/checkout@v3
-      - name: Install compile from source dependencies
-        run: |
-          conda install cmake pkg-config -c conda-forge
-      - name: Install test dependencies
-        run: |
-          python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu
-          # Ideally we would find a way to get those dependencies from pyproject.toml
-          python -m pip install numpy pytest pillow
-      - name: Install torchcodec from source, building against non-GPL FFmpeg
-        run: |
-          BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1 pip install -e ".[dev]" --no-build-isolation
       - name: Install ffmpeg, post build
         run: |
           # Ideally we would have checked for that before installing the wheel,
@@ -69,9 +98,30 @@ jobs:
 
           conda install "ffmpeg=${{ matrix.ffmpeg-version-for-tests }}" -c conda-forge
           ffmpeg -version
+
+      - name: Install test dependencies
+        run: |
+          python -m pip install --pre torchvision --index-url https://download.pytorch.org/whl/nightly/cpu
+          # Ideally we would find a way to get those dependencies from pyproject.toml
+          python -m pip install numpy pytest pillow
+
+      - name: Delete the src/ folder just for fun
+        run: |
+          # The only reason we checked-out the repo is to get access to the
+          # tests. We don't care about the rest. Out of precaution, we delete
+          # the src/ folder to be extra sure that we're running the code from
+          # the installed wheel rather than from the source.
+          # This is just to be extra cautious and very overkill because a)
+          # there's no way the `torchcodec` package from src/ can be found from
+          # the PythonPath: the main point of `src/` is precisely to protect
+          # against that and b) if we ever were to execute code from
+          # `src/torchcodec`, it would fail loudly because the built .so files
+          # aren't present there.
+          rm -r src/
+          ls
       - name: Smoke test
         run: |
           python test/decoders/manual_smoke_test.py
       - name: Run Python tests
         run: |
-          pytest test -vvv
+          pytest test

packaging/check_glibcxx.py

@@ -38,14 +38,15 @@
 
 MAX_ALLOWED = (3, 4, 19)
 
+libraries = sys.argv[1].split("\n")
 all_symbols = set()
-for line in sys.argv[1].split("\n"):
+for line in libraries:
     # We search for GLIBCXX_major.minor.micro
     if match := re.search(r"GLIBCXX_\d+\.\d+\.\d+", line):
         all_symbols.add(match.group(0))
 
 if not all_symbols:
-    raise ValueError("No GLIBCXX symbols found. Something is wrong.")
+    raise ValueError(f"No GLIBCXX symbols found in {libraries}. Something is wrong.")
 
 all_versions = (symbol.split("_")[1].split(".") for symbol in all_symbols)
 all_versions = (tuple(int(v) for v in version) for version in all_versions)

packaging/post_build_script.sh

@@ -1,38 +1,52 @@
 #!/bin/bash
 
+set -eux
+
 source packaging/helpers.sh
 
 wheel_path=$(pwd)/$(find dist -type f -name "*.whl")
 echo "Wheel content:"
 unzip -l $wheel_path
 
+unamestr=$(uname)
+if [[ "$unamestr" == 'Linux' ]]; then
+    ext="so"
+elif [[ "$unamestr" == 'Darwin' ]]; then
+    ext="dylib"
+else
+    echo "Unknown operating system: $unamestr"
+    exit 1
+fi
+
 for ffmpeg_major_version in 4 5 6 7; do
-    assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.so
+    assert_in_wheel $wheel_path torchcodec/libtorchcodec${ffmpeg_major_version}.${ext}
 done
-assert_not_in_wheel $wheel_path libtorchcodec.so
+assert_not_in_wheel $wheel_path libtorchcodec.${ext}
 
-for ffmpeg_so in libavcodec.so libavfilter.so libavformat.so libavutil.so libavdevice.so ; do
-    assert_not_in_wheel $wheel_path $ffmpeg_so
+for ffmpeg_ext in libavcodec.${ext} libavfilter.${ext} libavformat.${ext} libavutil.${ext} libavdevice.${ext} ; do
+    assert_not_in_wheel $wheel_path $ffmpeg_ext
 done
 
 assert_not_in_wheel $wheel_path "^test"
 assert_not_in_wheel $wheel_path "^doc"
 assert_not_in_wheel $wheel_path "^benchmarks"
 assert_not_in_wheel $wheel_path "^packaging"
 
-# See invoked python script below for details about this check.
-extracted_wheel_dir=$(mktemp -d)
-unzip -q $wheel_path -d $extracted_wheel_dir
-symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.)
-python packaging/check_glibcxx.py "$symbols_matches"
-
-echo "ls dist"
-ls dist
-
-old="linux_x86_64"
-new="manylinux_2_17_x86_64.manylinux2014_x86_64"
-echo "Replacing ${old} with ${new} in wheel name"
-mv dist/*${old}*.whl $(echo dist/*${old}*.whl | sed "s/${old}/${new}/")
+if [[ "$unamestr" == 'Linux' ]]; then
+    # See invoked python script below for details about this check.
+    extracted_wheel_dir=$(mktemp -d)
+    unzip -q $wheel_path -d $extracted_wheel_dir
+    symbols_matches=$(find $extracted_wheel_dir | grep ".so$" | xargs objdump --syms | grep GLIBCXX_3.4.)
+    python packaging/check_glibcxx.py "$symbols_matches"
+
+    echo "ls dist"
+    ls dist
+
+    old="linux_x86_64"
+    new="manylinux_2_17_x86_64.manylinux2014_x86_64"
+    echo "Replacing ${old} with ${new} in wheel name"
+    mv dist/*${old}*.whl $(echo dist/*${old}*.whl | sed "s/${old}/${new}/")
+fi
 
 echo "ls dist"
 ls dist

packaging/wheel_env.sh

@@ -0,0 +1 @@
+export BUILD_AGAINST_ALL_FFMPEG_FROM_S3=1