Upgrade to Python 3.12 (#1516)

we're bumping our image to Python 3.12

which required the following:
remove numpy-mkl: unfortunately we were not able to find/install a
compatible version, we opted to remove it based on previous conv we had
on this topic. we will instead use the default installed in colab.

remove cuml installation hack: 
thankfully are able to use the pre-installed base image version without
build errors.

unpinned package due to learn:
Learn is no longer dependent on this build, we can freely unpinned many
packages
-seaborn, scikit-learn, matplotlib, geopandas, TPOT, shapely, tfdf, ydf,
etc

remove incompatible packages:
Some of these are no longer support and cause build issues
-pydegensac, pymc3, eli5, etc

remove preinstalled package:
where applicable we removed packages that are already installed in colab
base image


https://b.corp.google.com/issues/468103319

Author: calderjo

Dockerfile.tmpl

@@ -12,9 +12,6 @@ RUN pip freeze | grep -E 'tensorflow|keras|torch|jax' > /colab_requirements.txt
 RUN cat /colab_requirements.txt >> /requirements.txt
 RUN cat /kaggle_requirements.txt >> /requirements.txt
 
-# TODO: GPU requirements.txt
-# TODO: merge them better (override matching ones).
-
 # Install Kaggle packages
 RUN uv pip install --system -r /requirements.txt
 
@@ -29,36 +26,25 @@ RUN uv pip install --system --force-reinstall --prerelease=allow "kagglehub[pand
 # to avoid affecting the larger build, we'll post-install it.
 RUN uv pip install --no-build-isolation --system "git+https://github.com/Kaggle/learntools"
 
-# b/408281617: Torch is adamant that it can not install cudnn 9.3.x, only 9.1.x, but Tensorflow can only support 9.3.x.
-# This conflict causes a number of package downgrades, which are handled in this command
-RUN uv pip install \
-    --index-url https://pypi.nvidia.com --extra-index-url https://pypi.org/simple/ --index-strategy unsafe-first-match \
-    --system --force-reinstall "cuml-cu12==25.2.1" \
-    "nvidia-cudnn-cu12==9.3.0.75" "nvidia-cublas-cu12==12.5.3.2" "nvidia-cusolver-cu12==11.6.3.83" \
-    "nvidia-cuda-cupti-cu12==12.5.82" "nvidia-cuda-nvrtc-cu12==12.5.82" "nvidia-cuda-runtime-cu12==12.5.82" \
-    "nvidia-cufft-cu12==11.2.3.61" "nvidia-curand-cu12==10.3.6.82" "nvidia-cusparse-cu12==12.5.1.3" \
-    "nvidia-nvjitlink-cu12==12.5.82"
-RUN uv pip install --system --force-reinstall "pynvjitlink-cu12==0.5.2"
-
-# b/385145217 Latest Colab lacks mkl numpy, install it.
-RUN uv pip install --system --force-reinstall -i https://pypi.anaconda.org/intel/simple numpy
-
 # newer daal4py requires tbb>=2022, but libpysal is downgrading it for some reason
 RUN uv pip install --system "tbb>=2022" "libpysal==4.9.2"
 
 # b/404590350: Ray and torchtune have conflicting tune cli, we will prioritize torchtune.
 # b/415358158: Gensim removed from Colab image to upgrade scipy
 # b/456239669: remove huggingface-hub pin when pytorch-lighting and transformer are compatible
 # b/315753846: Unpin translate package, currently conflicts with adk 1.17.0
-RUN uv pip install --system --force-reinstall --no-deps torchtune gensim "scipy<=1.15.3" "huggingface-hub==0.36.0" "google-cloud-translate==3.12.1"
+# b/468379293: Unpin Pandas once cuml/cudf are compatible, version 3.0 causes issues
+# b/468383498: numpy will auto-upgrade to 2.4.x, which causes issues with numerous packages
+# b/468367647: Unpin protobuf, version greater than v5.29.5 causes issues with numerous packages
+RUN uv pip install --system --force-reinstall --no-deps torchtune gensim "scipy<=1.15.3" "huggingface-hub==0.36.0" "google-cloud-translate==3.12.1" "numpy==2.0.2" "pandas==2.2.2"
+RUN uv pip install --system --force-reinstall "protobuf==5.29.5"
 
 # Adding non-package dependencies:
 ADD clean-layer.sh  /tmp/clean-layer.sh
 ADD patches/nbconvert-extensions.tpl /opt/kaggle/nbconvert-extensions.tpl
 ADD patches/template_conf.json /opt/kaggle/conf.json
 
-# /opt/conda/lib/python3.11/site-packages
-ARG PACKAGE_PATH=/usr/local/lib/python3.11/dist-packages
+ARG PACKAGE_PATH=/usr/local/lib/python3.12/dist-packages
 
 # Install GPU-specific non-pip packages.
 {{ if eq .Accelerator "gpu" }}
@@ -168,7 +154,7 @@ ADD patches/kaggle_gcp.py \
 
 # Figure out why this is in a different place?
 # Found by doing a export PYTHONVERBOSE=1 and then running python and checking for where it looked for it.
-ADD patches/sitecustomize.py /usr/lib/python3.11/sitecustomize.py
+ADD patches/sitecustomize.py /usr/lib/python3.12/sitecustomize.py
 
 ARG GIT_COMMIT=unknown \
     BUILD_DATE=unknown

config.txt

@@ -1,4 +1,4 @@
 BASE_IMAGE=us-docker.pkg.dev/colab-images/public/runtime
-BASE_IMAGE_TAG=release-colab_20250725-060057_RC00
+BASE_IMAGE_TAG=release-colab-external_20251024-060052_RC00
 CUDA_MAJOR_VERSION=12
 CUDA_MINOR_VERSION=5

kaggle_requirements.txt

@@ -7,11 +7,9 @@ PyArabic
 PyUpSet
 Pympler
 Rtree
-shapely<2
+shapely
 SimpleITK
-# b/302136621: Fix eli5 import for learntools, newer version require scikit-learn > 1.3
-TPOT==0.12.1
-Theano
+TPOT
 Wand
 annoy
 arrow
@@ -29,21 +27,14 @@ deap
 dipy
 docker
 easyocr
-# b/302136621: Fix eli5 import for learntools
-eli5
 emoji
 fastcore
-# b/445960030: Requires a newer version of fastai than the currently used base image.
-# Remove when relying on a newer base image.
-fastai>=2.8.4
 fasttext
 featuretools
 fiona
 fury
 fuzzywuzzy
 geojson
-# geopandas > v0.14.4 breaks learn tools
-geopandas==v0.14.4
 gensim
 # b/443054743,b/455550872
 google-adk[a2a,eval]
@@ -81,7 +72,7 @@ libpysal<=4.9.2
 lime
 line_profiler
 mamba
-matplotlib<3.8
+matplotlib
 mlcrate
 mne
 mpld3
@@ -90,9 +81,7 @@ nbconvert==6.4.5
 nbdev
 nilearn
 olefile
-# b/445960030: Broken in 1.19.0. See https://github.com/onnx/onnx/issues/7249.
-# Fixed with https://github.com/onnx/onnx/pull/7254. Upgrade when version with fix is published.
-onnx==1.18.0
+onnx
 openslide-bin
 openslide-python
 optuna
@@ -107,11 +96,9 @@ preprocessing
 pudb
 pyLDAvis
 pycryptodome
-pydegensac
 pydicom
 pyemd
 pyexcel-ods
-pymc3
 pymongo
 pypdf
 pytesseract
@@ -123,32 +110,25 @@ qtconsole
 ray
 rgf-python
 s3fs
-# b/302136621: Fix eli5 import for learntools
-scikit-learn==1.2.2
+scikit-learn
 # Scikit-learn accelerated library for x86
 scikit-learn-intelex>=2023.0.1
 scikit-multilearn
 scikit-optimize
 scikit-plot
 scikit-surprise
-# Also pinning seaborn for learntools
-seaborn==0.12.2
+seaborn
 git+https://github.com/facebookresearch/segment-anything.git
-# b/329869023: shap 0.45.0 breaks learntools
-shap==0.44.1
+shap
 squarify
 tensorflow-cloud
 tensorflow-io
 tensorflow-text
-tensorflow_decision_forests
 torchinfo
 torchmetrics
 torchtune
 transformers>=4.51.0
 vtk
 wavio
-# b/350573866: xgboost v2.1.0 breaks learntools
-xgboost==2.0.3
 xvfbwrapper
 ydata-profiling
-ydf

tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/2/metadata.json

@@ -1,5 +1,5 @@
 {
-    "module": "keras_nlp.src.models.bert.bert_backbone",
+    "module": "keras_hub.src.models.bert.bert_backbone",
     "class_name": "BertBackbone",
     "config": {
         "name": "bert_backbone",
@@ -13,7 +13,5 @@
         "max_sequence_length": 512,
         "num_segments": 2
     },
-    "registered_name": "keras_nlp>BertBackbone",
-    "assets": [],
-    "weights": "model.weights.h5"
+    "registered_name": "keras_hub>BertBackbone"
 }

tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/2/tokenizer.json

@@ -0,0 +1,10 @@
+{
+    "keras_version": "3.7.0",
+    "keras_hub_version": "0.19.0",
+    "parameter_count": 4385920,
+    "date_saved": "2024-12-20@19:42:50",
+    "tasks": [
+        "MaskedLM",
+        "TextClassifier"
+    ]
+}

…ncased/2/assets/tokenizer/vocabulary.txt …ncased/3/assets/tokenizer/vocabulary.txttests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/2/assets/tokenizer/vocabulary.txt renamed to tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/3/assets/tokenizer/vocabulary.txt tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/2/assets/tokenizer/vocabulary.txt renamed to tests/data/kagglehub/models/keras/bert/keras/bert_tiny_en_uncased/3/assets/tokenizer/vocabulary.txt

@@ -0,0 +1,27 @@
+{
+    "module": "keras_hub.src.models.bert.bert_tokenizer",
+    "class_name": "BertTokenizer",
+    "config": {
+        "name": "bert_tokenizer",
+        "trainable": true,
+        "dtype": {
+            "module": "keras",
+            "class_name": "DTypePolicy",
+            "config": {
+                "name": "int32"
+            },
+            "registered_name": null
+        },
+        "config_file": "tokenizer.json",
+        "vocabulary": null,
+        "sequence_length": null,
+        "lowercase": true,
+        "strip_accents": false,
+        "split": true,
+        "suffix_indicator": "##",
+        "oov_token": "[UNK]",
+        "special_tokens": null,
+        "special_tokens_in_strings": false
+    },
+    "registered_name": "keras_hub>BertTokenizer"
+}