Merge pull request #772 from Open-EO/761-apex-reference-check-needs-better-representation-ascii-art-diff-diff-image-statistics

761-apex-reference-check-needs-better-representation-ascii-art-diff-diff-image-statistics

Author: dsamaey

openeo/testing/results.py

@@ -134,21 +134,25 @@ def _compare_xarray_dataarray_xy(
     :return: list of issues (empty if no issues)
     """
     issues = []
-    threshold = abs(expected * rtol) + atol
-    diff_exact = abs(expected - actual)
+
+    actual_as_float = actual.astype(dtype=float)
+    expected_as_float = expected.astype(dtype=float)
+
+    threshold = abs(expected_as_float * rtol) + atol
+    diff_exact = abs(expected_as_float - actual_as_float)
     diff_mask = diff_exact > threshold
     diff_lenient = diff_exact.where(diff_mask)
 
-    non_x_y_dims = list(set(expected.dims) - {"x", "y"})
-    value_mapping = dict(map(lambda d: (d, expected[d].data), non_x_y_dims))
+    non_x_y_dims = list(set(expected_as_float.dims) - {"x", "y"})
+    value_mapping = dict(map(lambda d: (d, expected_as_float[d].data), non_x_y_dims))
     shape = tuple([len(value_mapping[x]) for x in non_x_y_dims])
 
     for shape_index, v in np.ndenumerate(np.ndarray(shape)):
         indexers = {}
         for index, value_index in enumerate(shape_index):
             indexers[non_x_y_dims[index]] = value_mapping[non_x_y_dims[index]][value_index]
         diff_data = diff_lenient.sel(indexers=indexers)
-        total_pixel_count = expected.sel(indexers).count().item()
+        total_pixel_count = expected_as_float.sel(indexers).count().item()
         diff_pixel_count = diff_data.count().item()
 
         if diff_pixel_count > 0:
@@ -230,8 +234,10 @@ def _compare_xarray_dataarray(
     except AssertionError as e:
         # TODO: message of `assert_allclose` is typically multiline, split it again or make it one line?
         issues.append(str(e).strip())
-    if compatible and {"x", "y"} <= set(expected.dims):
-        issues.extend(_compare_xarray_dataarray_xy(actual=actual, expected=expected, rtol=rtol, atol=atol, name=name))
+        if compatible and {"x", "y"} <= set(expected.dims):
+            issues.extend(
+                _compare_xarray_dataarray_xy(actual=actual, expected=expected, rtol=rtol, atol=atol, name=name)
+            )
     return issues
 
 

tests/testing/test_results.py

@@ -377,7 +377,7 @@ def test_allclose_xy_success(self, tmp_path, actual_dir, expected_dir):
         actual_ds.to_netcdf(actual_dir / "data.nc")
         assert_job_results_allclose(actual=actual_dir, expected=expected_dir, tmp_path=tmp_path, rtol=1)
 
-    def test_allclose_minimal_xy_different(self, tmp_path, actual_dir, expected_dir):
+    def test_allclose_xy_different(self, tmp_path, actual_dir, expected_dir):
         expected_ds = xarray.Dataset(
             {
                 "b1": xarray.Variable(dims=["t", "x", "y"], data=2 * numpy.ones((3, 4, 5))),
@@ -415,6 +415,71 @@ def test_allclose_minimal_xy_different(self, tmp_path, actual_dir, expected_dir)
         ):
             assert_job_results_allclose(actual=actual_dir, expected=expected_dir, tmp_path=tmp_path)
 
+    def test_allclose_bytes_minimal_xy_success(self, tmp_path, actual_dir, expected_dir):
+        expected_ds = xarray.Dataset(
+            {
+                "b1": xarray.Variable(dims=["t", "x", "y"], data=(2 * numpy.ones((3, 4, 5)))),
+                "b2": xarray.Variable(dims=["t", "x", "y"], data=(3 * numpy.ones((3, 4, 5)))),
+            },
+            coords={
+                "t": range(0, 3),
+                "x": range(4, 8),
+                "y": range(5, 10),
+            },
+        )
+        expected_ds.astype(dtype=numpy.dtype("B")).to_netcdf(expected_dir / "data.nc")
+        actual_ds = xarray.Dataset(
+            {
+                "b1": xarray.Variable(dims=["t", "x", "y"], data=(1 * numpy.ones((3, 4, 5)))),
+                "b2": xarray.Variable(dims=["t", "x", "y"], data=(3 * numpy.ones((3, 4, 5)))),
+            },
+            coords={
+                "t": range(0, 3),
+                "x": range(4, 8),
+                "y": range(5, 10),
+            },
+        )
+        actual_ds.astype(dtype=numpy.dtype("B")).to_netcdf(actual_dir / "data.nc")
+        assert_job_results_allclose(actual=actual_dir, expected=expected_dir, tmp_path=tmp_path, atol=2)
+
+    def test_allclose_bytes_minimal_xy_different(self, tmp_path, actual_dir, expected_dir):
+        expected_ds = xarray.Dataset(
+            {
+                "b1": xarray.Variable(dims=["t", "x", "y"], data=(2 * numpy.ones((3, 4, 5)))),
+                "b2": xarray.Variable(dims=["t", "x", "y"], data=(3 * numpy.ones((3, 4, 5)))),
+            },
+            coords={
+                "t": range(0, 3),
+                "x": range(4, 8),
+                "y": range(5, 10),
+            },
+        )
+        expected_ds.astype(dtype=numpy.dtype("B")).to_netcdf(expected_dir / "data.nc")
+        actual_ds = xarray.Dataset(
+            {
+                "b1": xarray.Variable(dims=["t", "x", "y"], data=(1 * numpy.ones((3, 4, 5)))),
+                "b2": xarray.Variable(dims=["t", "x", "y"], data=(3 * numpy.ones((3, 4, 5)))),
+            },
+            coords={
+                "t": range(0, 3),
+                "x": range(4, 8),
+                "y": range(5, 10),
+            },
+        )
+        actual_ds.astype(dtype=numpy.dtype("B")).to_netcdf(actual_dir / "data.nc")
+        with raises_assertion_error_or_not(
+            r"Issues for file 'data.nc'.*"
+            r"Issues for variable 'b1'.*"
+            r"Left and right DataArray objects are not close.*Differing values:.*"
+            r"t 0: value difference exceeds tolerance \(rtol 1e-06, atol 0.2\), min:1.0, max: 1.0, mean: 1.0, var: 0.0.*"
+            r"t 0: differing pixels: 20/20 \(100.0%\), bbox \(\(4, 5\), \(7, 9\)\) - 100.0% of the area.*"
+            r"t 1: value difference exceeds tolerance \(rtol 1e-06, atol 0.2\), min:1.0, max: 1.0, mean: 1.0, var: 0.0.*"
+            r"t 1: differing pixels: 20/20 \(100.0%\), bbox \(\(4, 5\), \(7, 9\)\) - 100.0% of the area.*"
+            r"t 2: value difference exceeds tolerance \(rtol 1e-06, atol 0.2\), min:1.0, max: 1.0, mean: 1.0, var: 0.0.*"
+            r"t 2: differing pixels: 20/20 \(100.0%\), bbox \(\(4, 5\), \(7, 9\)\) - 100.0% of the area"
+        ):
+            assert_job_results_allclose(actual=actual_dir, expected=expected_dir, tmp_path=tmp_path, atol=0.2)
+
     def test_allclose_minimal_xy_different_small_area(self, tmp_path, actual_dir, expected_dir):
         expected_ds = xarray.Dataset(
             {