Merge pull request #13 from EOPF-Sample-Service/sentinel-3

Add support for Sentinel 3 OLCI and SLSTR products

Author: mario-winkler

CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and this project adheres to [Semantic Versioning](https://semver.org/).
 
+## [Unreleased]
+
+### Added
+
+- Add support for the following Sentinel 3 OLCI products: `S03OLCEFR`, `S03OLCERR`, `S03OLCLFR`, `S03OLCLRR` [#9](https://github.com/EOPF-Sample-Service/eopf-stac/issues/9)
+- Add support for the following Sentinel 3 SLSTR products: `S03SLSRBT`, `S03SLSLST` [#11](https://github.com/EOPF-Sample-Service/eopf-stac/issues/11)
+- Add tests [#10](https://github.com/EOPF-Sample-Service/eopf-stac/issues/10)
+
 ## [0.8.0] - 2025-04-15
 
 ### Added

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "eopf-stac"
-version = "0.8.0"
+version = "0.9.0"
 authors = [
     { name="Mario Winkler", email="[email protected]" }
 ]
@@ -21,6 +21,7 @@ dependencies = [
     "requests",
     "s3fs",
     "pystac",
+    "stactools-sentinel3",
     "stactools-sentinel2",
     "stactools-sentinel1",    
     "geojson",

src/eopf_stac/constants.py renamed to src/eopf_stac/common/constants.py

@@ -6,7 +6,7 @@
 from pystac.link import Link
 from pystac.provider import ProviderRole
 
-from eopf_stac.eopf_xarray import EopfXarrayBackendConfig, OpMode
+from eopf_stac.common.eopf_xarray import EopfXarrayBackendConfig, OpMode
 
 SUPPORTED_PRODUCT_TYPES_S1 = [
     "S01SIWGRH",
@@ -22,7 +22,38 @@
     "S01SWVOCN",
 ]
 SUPPORTED_PRODUCT_TYPES_S2 = ["S02MSIL1C", "S02MSIL2A"]
-SUPPORTED_PRODUCT_TYPES_S3 = []
+
+SUPPORTED_S3_OLCI_L1_PRODUCT_TYPES = ["S03OLCEFR", "S03OLCERR"]
+SUPPORTED_S3_OLCI_L2_PRODUCT_TYPES = ["S03OLCLFR", "S03OLCLRR"]
+SUPPORTED_S3_SLSTR_L1_PRODUCT_TYPES = ["S03SLSRBT"]
+SUPPORTED_S3_SLSTR_L2_LST_PRODUCT_TYPE = ["S03SLSLST"]
+# SUPPORTED_S3_SLSTR_L2_FRP_PRODUCT_TYPE = ["S03SLSFRP"]  # conversion error
+SUPPORTED_PRODUCT_TYPES_S3 = (
+    SUPPORTED_S3_OLCI_L1_PRODUCT_TYPES
+    + SUPPORTED_S3_OLCI_L2_PRODUCT_TYPES
+    + SUPPORTED_S3_SLSTR_L1_PRODUCT_TYPES
+    + SUPPORTED_S3_SLSTR_L2_LST_PRODUCT_TYPE
+)
+
+# Other Sentinen-3 product types to support
+SUPPORTED_S3_SRAL_L1_PRODUCT_TYPES = ["S03AHRL1B"]  # sentinel-3-sra-l1b
+SUPPORTED_S3_SRAL_L2_PRODUCT_TYPES = ["S03AHRL2H"]  # sentinel-3-sra-l2-lan-hy
+SUPPORTED_S3_SYN_L2_PRODUCT_TYPES = [
+    "S03SYNAOD",
+    "S03SYNSDR",
+    "S03SYNV10",
+    "S03SYNVG1",
+    "S03SYNVGK",
+    "S03SYNVGP",
+]  # sentinel-3-syn-l2-aod, sentinel-3-syn-l2, sentinel-3-syn-l2-v10, sentinel-3-syn-l2-vg1, ?, sentinel-3-syn-l2-vgp
+
+
+# other SRAL listed in [1]
+#   - S03AHRL1A (SR_1_SRA_A_), S03ALRL1A (SR_1_SRA_A_), S03ALRL1B (SR_1_SRA_BS), S03ALRL2H (SR_2_LAN_HY)
+# other SYN listed in [1]: S03SYNMIS
+
+# [1] https://cpm.pages.eopf.copernicus.eu/eopf-cpm/main/PSFD/3-product-types-naming-rules.html
+
 PRODUCT_TYPE_TO_COLLECTION: Final[dict] = {
     "S01SIWGRH": "sentinel-1-l1-grd",
     "S01SSMGRH": "sentinel-1-l1-grd",
@@ -37,6 +68,13 @@
     "S01SWVOCN": "sentinel-1-l2-ocn",
     "S02MSIL1C": "sentinel-2-l1c",
     "S02MSIL2A": "sentinel-2-l2a",
+    "S03OLCEFR": "sentinel-3-olci-l1-efr",
+    "S03OLCERR": "sentinel-3-olci-l1-err",
+    "S03OLCLFR": "sentinel-3-olci-l2-lfr",
+    "S03OLCLRR": "sentinel-3-olci-l2-lrr",
+    "S03SLSRBT": "sentinel-3-slstr-l1-rbt",
+    "S03SLSFRP": "sentinel-3-slstr-l2-frp",
+    "S03SLSLST": "sentinel-3-slstr-l2-lst",
 }
 
 MEDIA_TYPE_ZARR = "application/vnd+zarr"
@@ -72,6 +110,13 @@
     roles=[ProviderRole.HOST, ProviderRole.PROCESSOR],
 )
 
+THUMBNAIL_ASSET: pystac.Asset = pystac.Asset(
+    href="",
+    title="",
+    media_type="image/png",
+    roles=["thumbnail"],
+)
+
 PRODUCT_METADATA_PATH: Final[str] = ".zmetadata"
 PRODUCT_METADATA_ASSET_KEY: Final[str] = "product_metadata"
 
@@ -102,3 +147,8 @@ def get_item_asset_product():
         roles=[ROLE_DATA, ROLE_METADATA],
         extra_fields=deepcopy(PRODUCT_ASSET_EXTRA_FIELDS),
     )
+
+
+PRODUCT_EXTENSION_SCHEMA_URI = "https://stac-extensions.github.io/product/v0.1.0/schema.json"
+PROCESSING_EXTENSION_SCHEMA_URI = "https://stac-extensions.github.io/processing/v1.2.0/schema.json"
+EOPF_EXTENSION_SCHEMA_URI = "https://cs-si.github.io/eopf-stac-extension/v1.2.0/schema.json"

src/eopf_stac/eopf_xarray.py renamed to src/eopf_stac/common/eopf_xarray.py

@@ -0,0 +1,209 @@
+import os
+
+import pystac
+from pystac.extensions.eo import EOExtension
+from pystac.extensions.sat import OrbitState, SatExtension
+from pystac.extensions.timestamps import TimestampsExtension
+from pystac.utils import now_in_utc, str_to_datetime
+
+from eopf_stac.common.constants import (
+    EOPF_EXTENSION_SCHEMA_URI,
+    PROCESSING_EXTENSION_SCHEMA_URI,
+    PRODUCT_EXTENSION_SCHEMA_URI,
+)
+
+
+def validate_metadata(metadata: dict) -> dict:
+    stac_discovery = metadata.get("metadata", {}).get(".zattrs", {}).get("stac_discovery")
+    if stac_discovery is None:
+        raise ValueError("JSON object 'stac_discovery' not found in .zmetadata file")
+
+    other_metadata = metadata.get("metadata", {}).get(".zattrs", {}).get("other_metadata")
+    if other_metadata is None:
+        raise ValueError("JSON object 'other_metadata' not found in .zmetadata file")
+
+    return metadata["metadata"]
+
+
+def rearrange_bbox(bbox):
+    longitudes = [bbox[0], bbox[2]]
+    latitudes = [bbox[1], bbox[3]]
+
+    corrected_bbox = [min(longitudes), min(latitudes), max(longitudes), max(latitudes)]
+    return corrected_bbox
+
+
+def get_identifier(stac_discovery: dict):
+    item_id = stac_discovery.get("id")
+    # CPM workaround for https://gitlab.eopf.copernicus.eu/cpm/eopf-cpm/-/issues/690
+    if item_id.lower().endswith(".safe") or item_id.lower().endswith(".sen3"):
+        item_id = os.path.splitext(item_id)[0]
+    return item_id
+
+
+def get_datetimes(properties: dict):
+    datetime = None
+    start_datetime = None
+    end_datetime = None
+    datetime_str = properties.get("datetime")
+    if datetime_str is not None:
+        # CPM workaround for https://gitlab.eopf.copernicus.eu/cpm/eopf-cpm/-/issues/643
+        if datetime_str == "null":
+            datetime = None
+        else:
+            datetime = str_to_datetime(datetime_str)
+
+    if datetime is None:
+        # start_datetime and end_datetime must be supplied
+        start_datetime_str = properties.get("start_datetime")
+        if start_datetime_str is not None:
+            start_datetime = str_to_datetime(start_datetime_str)
+            datetime = start_datetime
+        end_datetime_str = properties.get("end_datetime")
+        if end_datetime_str is not None:
+            end_datetime = str_to_datetime(end_datetime_str)
+
+    return (datetime, start_datetime, end_datetime)
+
+
+def fill_timestamp_properties(item: pystac.Item, properties: dict) -> None:
+    created_datetime = properties.get("created")
+    if created_datetime is None:
+        created_datetime = now_in_utc()
+    else:
+        created_datetime = str_to_datetime(created_datetime)
+    item.common_metadata.created = created_datetime
+    item.common_metadata.updated = created_datetime
+
+    ts_ext = TimestampsExtension.ext(item, add_if_missing=True)
+    ts_ext.apply(published=created_datetime)
+
+
+def fill_sat_properties(item: pystac.Item, properties: dict) -> None:
+    orbit_state = properties.get("sat:orbit_state")
+    abs_orbit = properties.get("sat:absolute_orbit")
+    rel_orbit = properties.get("sat:relative_orbit")
+    anx_datetime = properties.get("sat:anx_datetime")
+    platform_international_designator = properties.get("sat:platform_international_designator")
+
+    if any_not_none([orbit_state, abs_orbit, rel_orbit, anx_datetime, platform_international_designator]):
+        sat_ext = SatExtension.ext(item, add_if_missing=True)
+        if orbit_state:
+            sat_ext.orbit_state = OrbitState(orbit_state.lower())
+        if abs_orbit:
+            sat_ext.absolute_orbit = int(abs_orbit)
+        if rel_orbit:
+            sat_ext.relative_orbit = int(rel_orbit)
+        if anx_datetime:
+            sat_ext.anx_datetime = str_to_datetime(anx_datetime)
+        if platform_international_designator:
+            sat_ext.platform_international_designator = platform_international_designator
+
+
+def fill_eo_properties(item: pystac.Item, properties: dict) -> None:
+    cloud_cover = properties.get("eo:cloud_cover")
+    snow_cover = properties.get("eo:snow_cover")
+
+    if any_not_none([cloud_cover, snow_cover]):
+        eo = EOExtension.ext(item, add_if_missing=True)
+        if cloud_cover is not None:
+            eo.cloud_cover = cloud_cover
+        if snow_cover is not None:
+            eo.snow_cover = snow_cover
+
+
+def fill_processing_properties(item: pystac.Item, properties: dict) -> None:
+    # CPM workaround: following invalid values are ignored:
+    # "processing:expression": "systematic",
+    # "processing:facility": "OPE,OPE,OPE",
+    # "processing:version": "",
+
+    proc_expression = properties.get("processing:expression")
+    proc_lineage = properties.get("processing:lineage")
+    proc_level = properties.get("processing:level")
+    proc_facility = properties.get("processing:facility")
+    proc_datetime = properties.get("processing:datetime")
+    proc_version = properties.get("processing:version")
+    proc_software = properties.get("processing:software")
+    if any_not_none(
+        [proc_expression, proc_facility, proc_level, proc_lineage, proc_software, proc_datetime, proc_version]
+    ):
+        item.stac_extensions.append(PROCESSING_EXTENSION_SCHEMA_URI)
+        if proc_expression is not None and proc_expression != "systematic":
+            item.properties["processing:expression"] = proc_expression
+        if proc_software is not None:
+            item.properties["processing:software"] = proc_software
+        if proc_datetime is not None:
+            item.properties["processing:datetime"] = proc_datetime
+        if is_valid_string(proc_facility) and proc_facility != "OPE,OPE,OPE":
+            item.properties["processing:facility"] = proc_facility
+        if is_valid_string(proc_level):
+            item.properties["processing:level"] = proc_level
+        if is_valid_string(proc_lineage):
+            item.properties["processing:lineage"] = proc_lineage
+        if is_valid_string(proc_version):
+            item.properties["processing:version"] = proc_version
+
+
+def fill_product_properties(item: pystac.Item, product_type: str, properties: dict) -> None:
+    product_timeliness = properties.get("product:timeliness")
+    product_timeliness_category = properties.get("product:timeliness_category")
+    product_acquisition_type = properties.get("product:acquisition_type")
+    if any([product_type, product_acquisition_type, all([product_timeliness, product_timeliness_category])]):
+        item.stac_extensions.append(PRODUCT_EXTENSION_SCHEMA_URI)
+        if is_valid_string(product_type):
+            item.properties["product:type"] = product_type
+        if is_valid_string(product_acquisition_type):
+            item.properties["product:acquisition_type"] = product_acquisition_type
+        if all([is_valid_string(product_timeliness), is_valid_string(product_timeliness_category)]):
+            # CPM workaround for https://gitlab.eopf.copernicus.eu/cpm/eopf-cpm/-/issues/706
+            if product_timeliness != "MISSING":
+                item.properties["product:timeliness"] = product_timeliness
+                item.properties["product:timeliness_category"] = product_timeliness_category
+
+
+def fill_eopf_properties(item: pystac.Item, properties: dict) -> None:
+    """Fills the item with values of the EOPF STAC extension
+    See also: https://github.com/CS-SI/eopf-stac-extension
+    """
+    datatake_id = properties.get("eopf:datatake_id")
+    # CPM workaround for https://gitlab.eopf.copernicus.eu/cpm/eopf-cpm/-/issues/689
+    if datatake_id is None:
+        datatake_id = properties.get("eopf:data_take_id")
+    datastrip_id = properties.get("eopf:datastrip_id")
+    instrument_mode = properties.get("eopf:instrument_mode")
+    origin_datetime = properties.get("eopf:origin_datetime")
+    instrument_configuration_id = properties.get("eopf:instrument_configuration_id")
+
+    if any_not_none(
+        [
+            datatake_id,
+            datastrip_id,
+            instrument_mode,
+            origin_datetime,
+            instrument_configuration_id,
+        ]
+    ):
+        item.stac_extensions.append(EOPF_EXTENSION_SCHEMA_URI)
+        if is_valid_string(datatake_id):
+            item.properties["eopf:datatake_id"] = datatake_id
+        if is_valid_string(instrument_mode):
+            # CPM workaround
+            if instrument_mode != "Earth Observation":
+                item.properties["eopf:instrument_mode"] = instrument_mode
+        if origin_datetime:
+            item.properties["eopf:origin_datetime"] = origin_datetime
+        if is_valid_string(datastrip_id):
+            item.properties["eopf:datastrip_id"] = datastrip_id
+        if instrument_configuration_id is not None:
+            item.properties["eopf:instrument_configuration_id"] = instrument_configuration_id
+
+
+def is_valid_string(value: str) -> bool:
+    return value is not None and len(value) > 0
+
+
+def any_not_none(values: list) -> bool:
+    for v in values:
+        if v is not None:
+            return True