Merge pull request #8 from OKDP/update-docker-stacks

update Jupyter Docker environment with Spark 4, Python 3.12, and faster builds

Author: idirze

.build/.versions.yml

@@ -10,38 +10,58 @@ compatibility-matrix:
     java_version: [11]
     scala_version: [2.12, 2.13]
     hadoop_version: 3.2
-    spark_download_url: https://archive.apache.org/dist/spark/
+    # spark_download_url: https://archive.apache.org/dist/spark/
+    spark_download_url: https://github.com/OKDP/jupyterlab-docker/releases/download/spark-tarballs/
   # Maximum python version supported by spark-3.3.x: 3.10
   # Java support: 8/11/17
   - python_version: '3.10'
     spark_version: [3.3.1, 3.3.2, 3.3.3, 3.3.4]
     java_version: [17]
     scala_version: [2.12, 2.13]
     hadoop_version: 3
-    spark_download_url: https://archive.apache.org/dist/spark/
+    # spark_download_url: https://archive.apache.org/dist/spark/
+    spark_download_url: https://github.com/OKDP/jupyterlab-docker/releases/download/spark-tarballs/
   # Maximum python version supported by spark-3.4.x: 3.11
   # Java support: 8/11/17
   - python_version: 3.11
-    spark_version: [3.4.1, 3.4.2]
+    spark_version: [3.4.1, 3.4.2, 3.4.3, 3.4.4]
     java_version: [17]
     scala_version: [2.12, 2.13]
     hadoop_version: 3
-    spark_download_url: https://archive.apache.org/dist/spark/
+    # spark_download_url: https://archive.apache.org/dist/spark/
+    spark_download_url: https://github.com/OKDP/jupyterlab-docker/releases/download/spark-tarballs/
   # https://spark.apache.org/releases/spark-release-3-5-0.html
   # Minimum supported java version: 17/21
   - python_version: 3.11
-    spark_version: [3.5.1]
+    spark_version: [3.5.1, 3.5.2, 3.5.3, 3.5.4, 3.5.5, 3.5.6]
     java_version: [17]
     scala_version: [2.12, 2.13]
     hadoop_version: 3
-    spark_download_url: https://archive.apache.org/dist/spark/
+    # spark_download_url: https://archive.apache.org/dist/spark/
+    spark_download_url: https://github.com/OKDP/jupyterlab-docker/releases/download/spark-tarballs/
+  # Minimum supported java version: 17/21
+  # Support was dropped for 2.12 since spark 4
+  # Maximum python version supported by spark-4.0.x: 3.13
+  - python_version: 3.12
+    spark_version: [4.0.0, 4.0.1]
+    java_version: [17]
+    scala_version: [2.13]
+    hadoop_version: 3
+    # spark_download_url: https://archive.apache.org/dist/spark/
+    spark_download_url: https://github.com/OKDP/jupyterlab-docker/releases/download/spark-tarballs/
+  - python_version: 3.13
+    spark_version: [4.0.0, 4.0.1]
+    java_version: [17]
+    scala_version: [2.13]
+    hadoop_version: 3
+    # spark_download_url: https://archive.apache.org/dist/spark/
+    spark_download_url: https://github.com/OKDP/jupyterlab-docker/releases/download/spark-tarballs/
 ### Override the matrix above by providing the versions to build
 ### 1- The build-matrix is empty: build with all possible combintations
 ### 2- Override specific versions: build with all possible combinations which are compatible with that specific versions
 ### 3- The versions not present on compatibility-matrix are ignored
 build-matrix:
-  python_version: ['3.9', '3.10', '3.11']
-  spark_version: [3.2.4, 3.3.2, 3.3.4, 3.4.2, 3.5.1]
+  python_version: ['3.10', '3.11', '3.12']
+  spark_version: [3.3.4, 3.4.4, 3.5.6, 4.0.1]
   java_version: [11, 17]
-  scala_version: [2.12]
-
+  scala_version: [2.13]

.build/python/README.md

@@ -0,0 +1,8 @@
+# Run unit tests
+
+```shell
+
+cd .build/
+pytest -c python/pyproject.toml python/tests  -v --color=yes
+
+```

.build/python/okdp/extension/tagging/images_hierarchy.py

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["setuptools>=61", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "okdp"
+version = "0.1.0"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["okdp*", "okdp.extension*", "okdp.patches*"]
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]
+testpaths = ["tests"]