{ "process_graph": { "vectorbuffer1": { "process_id": "vector_buffer", "arguments": { "distance": 1280, "geometry": { "from_parameter": "geometry" }, "unit": "meter" } }, "dateshift1": { "process_id": "date_shift", "arguments": { "date": { "from_parameter": "startdate" }, "unit": "day", "value": -80 } }, "dateshift2": { "process_id": "date_shift", "arguments": { "date": { "from_parameter": "enddate" }, "unit": "day", "value": 80 } }, "loadcollection1": { "process_id": "load_collection", "arguments": { "bands": [ "VH", "VV" ], "id": "SENTINEL1_GRD", "properties": { "sat:orbit_state": { "process_graph": { "eq1": { "process_id": "eq", "arguments": { "x": { "from_parameter": "value" }, "y": "ASCENDING" }, "result": true } } }, "polarization": { "process_graph": { "eq2": { "process_id": "eq", "arguments": { "x": { "from_parameter": "value" }, "y": "VV&VH" }, "result": true } } } }, "spatial_extent": { "from_node": "vectorbuffer1" }, "temporal_extent": [ { "from_node": "dateshift1" }, { "from_node": "dateshift2" } ] } }, "sarbackscatter1": { "process_id": "sar_backscatter", "arguments": { "coefficient": "sigma0-ellipsoid", "contributing_area": false, "data": { "from_node": "loadcollection1" }, "elevation_model": "COPERNICUS_30", "ellipsoid_incidence_angle": false, "local_incidence_angle": false, "mask": false, "noise_removal": true } }, "renamelabels1": { "process_id": "rename_labels", "arguments": { "data": { "from_node": "sarbackscatter1" }, "dimension": "bands", "target": [ "VH_ASCENDING", "VV_ASCENDING" ] } }, "loadcollection2": { "process_id": "load_collection", "arguments": { "bands": [ "VH", "VV" ], "id": "SENTINEL1_GRD", "properties": { "sat:orbit_state": { "process_graph": { "eq3": { "process_id": "eq", "arguments": { "x": { "from_parameter": "value" }, "y": "DESCENDING" }, "result": true } } }, "polarization": { "process_graph": { "eq4": { "process_id": "eq", "arguments": { "x": { "from_parameter": "value" }, "y": "VV&VH" }, "result": true } } } }, "spatial_extent": { "from_node": "vectorbuffer1" }, "temporal_extent": [ { "from_node": "dateshift1" }, { "from_node": "dateshift2" } ] } }, "sarbackscatter2": { "process_id": "sar_backscatter", "arguments": { "coefficient": "sigma0-ellipsoid", "contributing_area": false, "data": { "from_node": "loadcollection2" }, "elevation_model": "COPERNICUS_30", "ellipsoid_incidence_angle": false, "local_incidence_angle": false, "mask": false, "noise_removal": true } }, "renamelabels2": { "process_id": "rename_labels", "arguments": { "data": { "from_node": "sarbackscatter2" }, "dimension": "bands", "target": [ "VH_DESCENDING", "VV_DESCENDING" ] } }, "mergecubes1": { "process_id": "merge_cubes", "arguments": { "cube1": { "from_node": "renamelabels1" }, "cube2": { "from_node": "renamelabels2" } } }, "loadcollection3": { "process_id": "load_collection", "arguments": { "bands": [ "B04", "B08" ], "id": "SENTINEL2_L2A", "properties": { "eo:cloud_cover": { "process_graph": { "lte1": { "process_id": "lte", "arguments": { "x": { "from_parameter": "value" }, "y": 95 }, "result": true } } } }, "spatial_extent": { "from_node": "vectorbuffer1" }, "temporal_extent": [ { "from_node": "dateshift1" }, { "from_node": "dateshift2" } ] } }, "loadcollection4": { "process_id": "load_collection", "arguments": { "bands": [ "SCL" ], "id": "SENTINEL2_L2A", "properties": { "eo:cloud_cover": { "process_graph": { "lte2": { "process_id": "lte", "arguments": { "x": { "from_parameter": "value" }, "y": 95 }, "result": true } } } }, "spatial_extent": { "from_node": "vectorbuffer1" }, "temporal_extent": [ { "from_node": "dateshift1" }, { "from_node": "dateshift2" } ] } }, "toscldilationmask1": { "process_id": "to_scl_dilation_mask", "arguments": { "data": { "from_node": "loadcollection4" }, "erosion_kernel_size": 3, "kernel1_size": 17, "kernel2_size": 77, "mask1_values": [ 2, 4, 5, 6, 7 ], "mask2_values": [ 3, 8, 9, 10, 11 ] } }, "mask1": { "process_id": "mask", "arguments": { "data": { "from_node": "loadcollection3" }, "mask": { "from_node": "toscldilationmask1" } } }, "ndvi1": { "process_id": "ndvi", "arguments": { "data": { "from_node": "mask1" }, "nir": "B08", "red": "B04", "target_band": "NDVI" } }, "filterbands1": { "process_id": "filter_bands", "arguments": { "bands": [ "NDVI" ], "data": { "from_node": "ndvi1" } } }, "biopar1": { "process_id": "biopar", "arguments": { "biopar_type": "FCOVER", "spatial_extent": { "from_node": "vectorbuffer1" }, "temporal_extent": [ { "from_node": "dateshift1" }, { "from_node": "dateshift2" } ] }, "namespace": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/main/algorithm_catalog/vito/biopar/openeo_udp/biopar.json" }, "filterbands2": { "process_id": "filter_bands", "arguments": { "bands": [ "FCOVER" ], "data": { "from_node": "biopar1" } } }, "biopar2": { "process_id": "biopar", "arguments": { "biopar_type": "FAPAR", "spatial_extent": { "from_node": "vectorbuffer1" }, "temporal_extent": [ { "from_node": "dateshift1" }, { "from_node": "dateshift2" } ] }, "namespace": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/main/algorithm_catalog/vito/biopar/openeo_udp/biopar.json" }, "filterbands3": { "process_id": "filter_bands", "arguments": { "bands": [ "FAPAR" ], "data": { "from_node": "biopar2" } } }, "eq5": { "process_id": "eq", "arguments": { "case_sensitive": false, "x": { "from_parameter": "output" }, "y": "fapar" } }, "if1": { "process_id": "if", "arguments": { "accept": { "from_node": "filterbands3" }, "reject": null, "value": { "from_node": "eq5" } } }, "eq6": { "process_id": "eq", "arguments": { "case_sensitive": false, "x": { "from_parameter": "output" }, "y": "fcover" } }, "if2": { "process_id": "if", "arguments": { "accept": { "from_node": "filterbands2" }, "reject": { "from_node": "if1" }, "value": { "from_node": "eq6" } } }, "mergecubes2": { "process_id": "merge_cubes", "arguments": { "cube1": { "from_node": "if2" }, "cube2": { "from_node": "filterbands1" } } }, "eq7": { "process_id": "eq", "arguments": { "case_sensitive": false, "x": { "from_parameter": "output" }, "y": "ndvi" } }, "if3": { "process_id": "if", "arguments": { "accept": { "from_node": "filterbands1" }, "reject": { "from_node": "mergecubes2" }, "value": { "from_node": "eq7" } } }, "resamplecubespatial1": { "process_id": "resample_cube_spatial", "arguments": { "data": { "from_node": "mergecubes1" }, "target": { "from_node": "if3" } } }, "mergecubes3": { "process_id": "merge_cubes", "arguments": { "cube1": { "from_node": "resamplecubespatial1" }, "cube2": { "from_node": "if3" } } }, "applyneighborhood1": { "process_id": "apply_neighborhood", "arguments": { "data": { "from_node": "mergecubes3" }, "overlap": [ { "dimension": "x", "value": 8, "unit": "px" }, { "dimension": "y", "value": 8, "unit": "px" } ], "process": { "process_graph": { "runudf1": { "process_id": "run_udf", "arguments": { "context": { "startdate": { "from_parameter": "startdate" }, "enddate": { "from_parameter": "enddate" }, "output": { "from_parameter": "output" } }, "data": { "from_parameter": "data" }, "runtime": "Python", "udf": "# /// script\n# dependencies = [\n# \"opencv-python-headless\",\n# ]\n# ///\n#!/usr/bin/env python\nimport contextlib as __stickytape_contextlib\n\n@__stickytape_contextlib.contextmanager\ndef __stickytape_temporary_dir():\n import tempfile\n import shutil\n dir_path = tempfile.mkdtemp()\n try:\n yield dir_path\n finally:\n shutil.rmtree(dir_path)\n\nwith __stickytape_temporary_dir() as __stickytape_working_dir:\n def __stickytape_write_module(path, contents):\n import os, os.path\n\n def make_package(path):\n parts = path.split(\"/\")\n partial_path = __stickytape_working_dir\n for part in parts:\n partial_path = os.path.join(partial_path, part)\n if not os.path.exists(partial_path):\n os.mkdir(partial_path)\n with open(os.path.join(partial_path, \"__init__.py\"), \"wb\") as f:\n f.write(b\"\\n\")\n\n make_package(os.path.dirname(path))\n\n full_path = os.path.join(__stickytape_working_dir, path)\n with open(full_path, \"wb\") as module_file:\n module_file.write(contents)\n\n import sys as __stickytape_sys\n __stickytape_sys.path.insert(0, __stickytape_working_dir)\n\n __stickytape_write_module('cropsar_px_openeo/__init__.py', b'from cropsar_px_openeo.config.config import Config\\n\\nconfig = Config()')\n __stickytape_write_module('cropsar_px_openeo/config/__init__.py', b'')\n __stickytape_write_module('cropsar_px_openeo/config/config.py', b'import configparser\\nimport importlib.resources\\nimport os\\nfrom logging import Logger\\n\\n\\nclass Config:\\n def __init__(self, environment=os.environ.get(\"CROPSAR_PX_ENV\", \"cdse\")):\\n self.logger = Logger(\"configparser\")\\n self.config = self._load_config(environment=environment)\\n\\n def _load_config(self, environment: str) -> configparser.ConfigParser:\\n \"\"\"\\n Load in the config file\\n :param environment: String representing the environment for which to load the config\\n :return:\\n \"\"\"\\n try:\\n path = importlib.resources.files(\"cropsar_px_openeo.resources.config\") / f\"{environment}.cfg\"\\n config = configparser.ConfigParser()\\n with path.open() as config_file:\\n config.read_file(config_file)\\n self.logger.info(f\"Readed configuration from {path}\")\\n except (AttributeError, Exception) as e:\\n config = None\\n self.logger.warning(f\"Could not find config file for environment {environment}\")\\n self.logger.warning(e)\\n return config\\n\\n def get_collection_id(self, collection_type: str) -> str:\\n \"\"\"\\n Retrieve the ID of the collection based on the provided type. This will be read from the [collections] section\\n within the configuration\\n :param collection_type: Type of the collection for which to get the ID\\n :return: String representing the ID of the openEO collection\\n \"\"\"\\n self.logger.debug(f\"Reading collection {collection_type} from configuration\")\\n return self.config[\"collections\"][collection_type]\\n\\n def get_polarization(self, polarization_type: str) -> str:\\n \"\"\"\\n Retrieve the polarization to use for Sentinel-1 data loading. This will be read from the [polarization] section\\n within the configuration\\n Returns:\\n String representing the polarization to use for Sentinel-1 data loading\\n \"\"\"\\n self.logger.debug(\"Reading polarization from configuration\")\\n value = self.config[\"polarization\"][polarization_type]\\n return value.strip()\\n\\n def get_openeo_url(self) -> str:\\n \"\"\"\\n Read the openEO URL from the config file\\n :return: URL of the openEO endpoint\\n \"\"\"\\n self.logger.debug(\"Reading openEO endpoint from configuration\")\\n return self.config[\"openeo\"][\"url\"]\\n\\n def get_openeo_credentials(self) -> dict:\\n \"\"\"\\n Read the openEO credentials from the config file\\n :return: Dictionary containing the \\'client_id\\', \\'client_secret\\' and \\'provider\\' that can be used to authenticate\\n with openEO\\n \"\"\"\\n self.logger.debug(\"Reading openEO endpoint from configuration\")\\n return {\\n \"id\": self.config[\"openeo\"][\"client_id\"],\\n \"secret\": self.config[\"openeo\"][\"client_secret\"],\\n \"provider\": self.config[\"openeo\"][\"provider\"],\\n }\\n\\n @staticmethod\\n def _get_namespace_url(url: str, namespace: str, service: str):\\n \"\"\"\\n Create the namespace URL to access a certain service from a namespace\\n :param url: Base openEO URL\\n :param namespace: Name of the namespace\\n :param service: ID of the service\\n :return: String representing the URL on which the service is accessible\\n \"\"\"\\n return f\"https://{url}/openeo/processes/{namespace}/{service}\"\\n\\n def get_service_info(self, service: str) -> dict:\\n \"\"\"\\n Read the `id` and `namespace` of a service from the config file\\n :param service: Name of the service from with to read the information\\n :return: Dictionary containing the `id` and `namespace` of the service\\n \"\"\"\\n self.logger.debug(f\"Looking up service information for {service}\")\\n key = f\"service_{service}\"\\n return {\\n \"id\": self.config[key][\"id\"],\\n \"namespace\": self._get_namespace_url(\\n url=self.config[\"openeo\"][\"url\"],\\n namespace=self.config[key][\"namespace\"],\\n service=self.config[key][\"id\"],\\n ),\\n }\\n\\n def get_udp_info(self) -> dict:\\n \"\"\"\\n Return the name and namespace of the UDP process to use for the CropSAR service\\n :return: Dictionary containing the `id` and `namespace` of the service\\n \"\"\"\\n self.logger.debug(\"Looking up the UDP process name\")\\n return {\\n \"id\": self.config[\"udp\"][\"process\"],\\n \"namespace\": self._get_namespace_url(\\n url=self.config[\"openeo\"][\"url\"],\\n namespace=self.config[\"udp\"][\"namespace\"],\\n service=self.config[\"udp\"][\"process\"],\\n ),\\n }\\n\\n def get_udp_summary(self) -> str:\\n \"\"\"\\n Return the summary of the UDP process to use for the CropSAR service\\n :return: String representing the summary of the CropSAR s ervice\\n \"\"\"\\n self.logger.debug(\"Looking up the UDP summary\")\\n return self.config[\"udp\"][\"summary\"]\\n\\n def get_udf_archives(self) -> list:\\n \"\"\"\\n Return the list of archives that should be included when executing the CropSAR UDP\\n :return: List of UDF archives\\n \"\"\"\\n self.logger.debug(\"Looking up the UDF archives\")\\n return [f\"{self.config[\\'udp\\'][\\'udf_archive\\']}#tmp/env/venv_cropsar\"]\\n')\n __stickytape_write_module('cropsar_px_openeo/udf/__init__.py', b'')\n __stickytape_write_module('cropsar_px_openeo/udf/cropsar_2d/__init__.py', b'')\n __stickytape_write_module('cropsar_px_openeo/udf/cropsar_2d/utils/__init__.py', b'')\n __stickytape_write_module('cropsar_px_openeo/udf/cropsar_2d/utils/env.py', b'import sys\\nfrom pathlib import Path\\n\\nfrom cropsar_px_openeo.utils.logger import Logger\\n\\nlogger = Logger(__name__, udf=True)\\n\\n\\ndef setup_env():\\n \"\"\"Add dependency paths for both py8_legacy and py11_onnx runtimes.\"\"\"\\n\\n candidates = []\\n\\n # py8_legacy: archive extracted to tmp/env/venv_cropsar\\n venv_root = Path(\"tmp/env/venv_cropsar\")\\n candidates.extend([\\n venv_root,\\n venv_root / \"site-packages\",\\n ])\\n candidates.extend(venv_root.glob(\"lib/python*/site-packages\"))\\n\\n # py11_onnx: explicit dependency archive target\\n candidates.append(Path(\"onnx_deps\"))\\n\\n added = []\\n for p in candidates:\\n p_str = str(p)\\n if p.exists() and p_str not in sys.path:\\n sys.path.insert(0, p_str)\\n added.append(p_str)\\n\\n if added:\\n logger.debug(f\"Added dependency paths to PYTHONPATH: {added}\")\\n else:\\n logger.warn(\"No dependency path found for setup_env().\")\\n')\n __stickytape_write_module('cropsar_px_openeo/utils/__init__.py', b'')\n __stickytape_write_module('cropsar_px_openeo/utils/logger.py', b'import logging\\nfrom typing import Any\\n\\nfrom openeo.udf import inspect\\n\\n\\nclass Logger:\\n \"\"\"\\n Custom logger instance to support default and\\n UDF logging (https://open-eo.github.io/openeo-python-client/udf.html#logging-from-a-udf)\\n \"\"\"\\n\\n def __init__(self, name: str, udf=False):\\n \"\"\"\\n Create a new logger instance\\n :param name: Name of the logger instance to use\\n :param udf: Flag indicating if the logger is used within a UDF (default: False)\\n \"\"\"\\n self.udf = udf\\n self.logger = logging.getLogger(name)\\n\\n def debug(self, message: str, data: Any = None):\\n self.logger.debug(message)\\n self._inspect(data, message, \"debug\")\\n\\n def info(self, message: str, data: Any = None):\\n self.logger.info(message)\\n self._inspect(data, message, \"info\")\\n\\n def warn(self, message: str, data: Any = None):\\n self.logger.warning(message)\\n self._inspect(data, message, \"warning\")\\n\\n def warning(self, message: str, data: Any = None):\\n self.warn(message=message, data=data)\\n\\n def error(self, message: str, data: Any = None):\\n self.logger.error(message)\\n self._inspect(data, message, \"error\")\\n\\n def _inspect(self, data: Any, message: str, level: str):\\n if self.udf:\\n inspect(data=data, message=message, level=level)\\n')\n __stickytape_write_module('cropsar_px_openeo/types.py', b'from typing import Literal\\n\\nOutput = Literal[\"NDVI\", \"FAPAR\", \"FCOVER\", \"RGB_NIR\"]\\nOrbitDirection = Literal[\"ASCENDING\", \"DESCENDING\"]\\n')\n __stickytape_write_module('cropsar_px_openeo/udf/constants.py', b'\"\"\"Constants for CropSAR UDF processing.\\n\\nOrganised in sections:\\n- Common: shared by both CropSAR 1D and 2D\\n- CropSAR 1D specific\\n- CropSAR 2D specific\\n- CropSAR 2D ONNX (py11) specific\\n\"\"\"\\n\\n# ---------------------------------------------------------------------------\\n# Common \\xe2\\x80\\x94 shared by CropSAR 1D and 2D\\n# ---------------------------------------------------------------------------\\nTEMPORAL_BUFFER = 80 # Extra days loaded around the requested range\\nTEMPORAL_FREQUENCY = \"5D\" # Output temporal resolution\\nSPATIAL_RESOLUTION = 10 # Pixel size in meters\\n\\nSENTINEL1_BANDS = (\"VH\", \"VV\")\\nORBIT_DIRECTIONS = (\"ASCENDING\", \"DESCENDING\")\\n\\nSENTINEL2_BANDS = {\"NDVI\": [\"NDVI\"], \"FAPAR\": [\"FAPAR\"], \"FCOVER\": [\"FCOVER\"]}\\nSENTINEL2_NDVI = \"NDVI\"\\n\\nDOWNLOAD_CHUNK_SIZE = 1024 * 1024\\nDOWNLOAD_TIMEOUT = 60\\n\\n# ---------------------------------------------------------------------------\\n# CropSAR 1D specific\\n# ---------------------------------------------------------------------------\\nINWARDS_BUFFER = 10 # Buffer inward in meters to avoid edge effects during spatial aggregation\\n\\n# ---------------------------------------------------------------------------\\n# CropSAR 2D specific\\n# ---------------------------------------------------------------------------\\nSPATIAL_WINDOW_SIZE = 128 # Spatial window (px) for 2D neighbourhood processing\\n\\n# Model zip archives downloaded by the py8_legacy UDF at executor startup\\nMODEL_URLS = {\\n \"NDVI\": \"https://artifactory.vgt.vito.be/artifactory/auxdata-public/cropsar_px/2.0.0/20230628T094900_cnn_transformer_multi_repr2_ndvi_only.zip\",\\n \"FAPAR\": \"https://artifactory.vgt.vito.be/artifactory/auxdata-public/cropsar_px/2.0.0/20230628T100340_cnn_transformer_multi_repr2_fapar_only.zip\",\\n \"FCOVER\": \"https://artifactory.vgt.vito.be/artifactory/auxdata-public/cropsar_px/2.0.0/20230629T081418_cnn_transformer_multi_repr2_fcover_only.zip\",\\n}\\n# ---------------------------------------------------------------------------\\n# Model files are extracted from cropsar2d_onnx.zip#cropsar2d_onnx_models\\nONNX_MODEL_DIR = \"cropsar2d_onnx_models/cropsar2d_onnx\"\\n\\nONNX_MODEL_PATHS = {\\n \"NDVI\": f\"{ONNX_MODEL_DIR}/cropsar_px_ndvi.onnx\",\\n \"FAPAR\": f\"{ONNX_MODEL_DIR}/cropsar_px_fapar.onnx\",\\n \"FCOVER\": f\"{ONNX_MODEL_DIR}/cropsar_px_fcover.onnx\",\\n}\\nSCALER_PATHS = {\\n \"NDVI\": f\"{ONNX_MODEL_DIR}/scaler_ndvi.json\",\\n \"FAPAR\": f\"{ONNX_MODEL_DIR}/scaler_fapar.json\",\\n \"FCOVER\": f\"{ONNX_MODEL_DIR}/scaler_fcover.json\",\\n}\\n\\nN_TS = 32 # Fixed temporal window (PositionalEncoder constraint)\\nFILL_NAN = 0 # Value to fill NaN in S2 (done inside ONNX model)\\nEDGE_REPR = True # All production models use edge representation\\nN_CHANNELS_OUT = 1 # One output channel per model\\nMODEL_RESOLUTION = 128 # Spatial resolution the model was trained on\\n')\n __stickytape_write_module('cropsar_px_openeo/udf/cropsar_2d/preprocess/__init__.py', b'')\n __stickytape_write_module('cropsar_px_openeo/udf/cropsar_2d/preprocess/s1.py', b'import numpy as np\\nimport pandas\\nimport xarray\\n\\nfrom cropsar_px_openeo.udf.constants import ORBIT_DIRECTIONS, SENTINEL1_BANDS\\nfrom cropsar_px_openeo.udf.cropsar_2d.preprocess import speckle\\nfrom cropsar_px_openeo.utils.logger import Logger\\n\\nlogger = Logger(__name__, udf=True)\\n\\n\\ndef prepare_s1(\\n array: xarray.DataArray,\\n temporal_index: pandas.DatetimeIndex,\\n) -> xarray.DataArray:\\n \"\"\"\\n Prepare the Sentinel-1 data to be used as input for the CropSAR_px model.\\n :param array: input data array\\n :param temporal_index: temporal index\\n :return: Sentinel-1 data in dimension order (t, bands, y, x)\\n \"\"\"\\n # select Sentinel-1 bands, input is in power units\\n s1 = filter_s1_bands(array)\\n s1 = multitemporal_speckle_filter(s1)\\n s1 = resample_s1(s1, temporal_index)\\n s1 = s1.transpose(\"t\", \"bands\", \"y\", \"x\")\\n return to_dB(s1)\\n\\n\\ndef to_dB(array: xarray.DataArray) -> xarray.DataArray:\\n return 10 * np.log10(array)\\n\\n\\ndef filter_s1_bands(array: xarray.DataArray) -> xarray.DataArray:\\n \"\"\"\\n Filter the Sentinel-1 bands from the input array.\\n If the input array contains Sentinel-1 bands for both orbit directions, the best one will be selected.\\n :param array: input data array\\n :return: filtered data array containing VV and VH bands\\n \"\"\"\\n if all(band in array.bands for band in SENTINEL1_BANDS):\\n logger.info(f\"Using provided Sentinel-1 bands {SENTINEL1_BANDS}\")\\n return array.sel(bands=list(SENTINEL1_BANDS))\\n\\n logger.info(\"Selecting best orbit direction for Sentinel-1 bands\")\\n # select one of the bands for both orbit directions\\n band_orbs = array.sel(bands=[f\"{SENTINEL1_BANDS[0]}_{orbit_direction}\" for orbit_direction in ORBIT_DIRECTIONS])\\n # count the number of valid pixels, take the one with the most\\n best_band = band_orbs.bands[band_orbs.count(dim=(\"x\", \"y\", \"t\")).argmax()].item()\\n # derive the direction from the band name\\n best_direction = best_band.split(\"_\")[-1]\\n logger.info(f\"Selecting {best_direction} orbit direction\")\\n # get the bands for the best orbit direction\\n s1 = array.sel(bands=[f\"{band}_{best_direction}\" for band in SENTINEL1_BANDS])\\n # rename them to VH and VV\\n s1[\"bands\"] = list(SENTINEL1_BANDS)\\n return s1\\n\\n\\ndef multitemporal_speckle_filter(array: xarray.DataArray) -> xarray.DataArray:\\n \"\"\"\\n Apply a multi-temporal speckle filter to the Sentinel-1 data in the input array.\\n :param array: Sentinel-1 input data\\n :return: Sentinel-1 data with multi-temporal speckle filter applied\\n \"\"\"\\n array = array.transpose(\"bands\", \"t\", \"y\", \"x\")\\n for band in array.bands:\\n data = array.loc[dict(bands=band)].values\\n # Speckle filtering uses 0 as nodata\\n idxnodata = np.isnan(data)\\n data[idxnodata] = 0\\n\\n filtered = np.rollaxis(\\n speckle.mtfilter(\\n np.rollaxis(data, 0, 3), # input shape for mtfilter: (rows, cols, t)\\n \"gamma\",\\n ),\\n 2,\\n 0,\\n ) # go back to shape: (t, rows, cols)\\n\\n filtered[idxnodata] = np.nan\\n array.loc[dict(bands=band)] = filtered\\n\\n return array\\n\\n\\ndef resample_s1(array: xarray.DataArray, temporal_index: pandas.DatetimeIndex) -> xarray.DataArray:\\n \"\"\"\\n Resample the Sentinel-1 data to 5-day frequency and interpolate NaN values.\\n :param array: Sentinel-1 data\\n :param temporal_index: temporal index with 5-day frequency\\n :return: resampled Sentinel-1 data\\n \"\"\"\\n return (\\n array.resample(t=temporal_index.freqstr)\\n .mean(skipna=True)\\n .interpolate_na(dim=\"t\", method=\"linear\")\\n .reindex(t=temporal_index, method=\"ffill\", tolerance=temporal_index.freqstr)\\n )\\n')\n __stickytape_write_module('cropsar_px_openeo/udf/cropsar_2d/preprocess/speckle.py', b'import numpy as np\\nfrom scipy import ndimage\\n\\n# based on https://git.vito.be/projects/LCLU/repos/satio/browse/satio/utils/speckle.py\\n\\n\\ndef mtfilter(stack, kernel, mtwin=7, enl=3):\\n \"\"\"\\n stack: np array with multi-temporal stack of backscatter images (linear\\n scale)\\n\\n kernel: \\'mean\\',\\'gauss\\',\\'gamma\\' - \\'gamma\\' is recommended (slower than the\\n other kernels though)\\n\\n mtwin: filter window size - recommended mtwin=7\\n\\n enl: only required for kernel \\'gamma\\' - recommended for S1 enl = 3\\n \"\"\"\\n rows, cols, layers = stack.shape\\n filtim = np.zeros((rows, cols, layers))\\n\\n rcs = image_sum = image_num = image_fil = None # pylance unbound warning\\n\\n for no in range(0, layers):\\n # Initiate arrays\\n if no == 0:\\n image_sum = np.zeros((rows, cols))\\n image_num = np.zeros((rows, cols))\\n image_fil = np.zeros((rows, cols, layers))\\n\\n if kernel == \"mean\":\\n rcs = ndimage.uniform_filter(stack[:, :, no], size=mtwin, mode=\"mirror\")\\n elif kernel == \"gauss\":\\n rcs = ndimage.gaussian_filter(stack[:, :, no], mtwin / 4, mode=\"mirror\")\\n elif kernel == \"gamma\":\\n rcs = GammaMAP(stack[:, :, no], mtwin, enl, 0)\\n\\n with np.errstate(divide=\"ignore\", invalid=\"ignore\"):\\n ratio = stack[:, :, no] / rcs\\n ratio[np.isnan(ratio)] = 0\\n\\n image_sum = image_sum + ratio\\n image_num = image_num + (ratio > 0)\\n image_fil[:, :, no] = rcs\\n\\n with np.errstate(invalid=\"ignore\"):\\n for no in range(0, layers):\\n im = stack[:, :, no]\\n filtim1 = image_fil[:, :, no] * image_sum / image_num\\n filtim1[np.isnan(filtim1)] = 0\\n fillmask = (filtim1 == 0) & (im > 0)\\n filtim1[fillmask] = im[fillmask]\\n mask = im > 0\\n filtim1[mask == 0] = im[mask == 0]\\n filtim[:, :, no] = filtim1\\n\\n return filtim\\n\\n\\ndef GammaMAP(band, size, ENL, ndv):\\n img = band\\n img[band == ndv] = 0.0\\n sig_v2 = 1.0 / ENL\\n ENL2 = ENL + 1.0\\n sfak = 1.0 + sig_v2\\n img_mean2 = ndimage.uniform_filter(pow(img, 2), size=size)\\n img_mean2[img == ndv] = 0.0\\n img_mean = ndimage.uniform_filter(img, size=size)\\n img_mean[img == ndv] = 0.0\\n var_z = img_mean2 - pow(img_mean, 2)\\n out = img_mean\\n\\n with np.errstate(divide=\"ignore\", invalid=\"ignore\"):\\n fact1 = var_z / pow(img_mean, 2)\\n fact1[np.isnan(fact1)] = 0\\n\\n mask = (fact1 > sig_v2) & ((var_z - pow(img_mean, 2) * sig_v2) > 0.0)\\n\\n if mask.any():\\n n = (pow(img_mean, 2) * sfak) / (var_z - pow(img_mean, 2) * sig_v2)\\n phalf = (img_mean * (ENL2 - n)) / (2 * n)\\n q = ENL * img_mean * img / n\\n out[mask] = -phalf[mask] + np.sqrt(pow(phalf[mask], 2) + q[mask])\\n\\n out[img == 0.0] = ndv\\n return out\\n')\n __stickytape_write_module('cropsar_px_openeo/udf/cropsar_2d/preprocess/s2.py', b'import numpy\\nimport pandas\\nimport xarray\\n\\nfrom cropsar_px_openeo.udf.constants import SENTINEL2_BANDS, SENTINEL2_NDVI\\nfrom cropsar_px_openeo.udf.cropsar_2d.preprocess.filter_dips import flaglocalminima\\nfrom cropsar_px_openeo.utils.logger import Logger\\n\\nlogger = Logger(__name__, udf=True)\\n\\n\\ndef _log_nan_stats(stage: str, arr: xarray.DataArray) -> None:\\n values = arr.values\\n total = int(values.size)\\n valid = int(numpy.count_nonzero(~numpy.isnan(values)))\\n logger.error(\\n f\"S2 diagnostics [{stage}] shape={values.shape}, dims={arr.dims}, \"\\n f\"valid={valid}/{total} ({(100.0 * valid / max(total, 1)):.2f}%)\"\\n )\\n\\n if \"t\" in arr.dims:\\n t_axis = arr.get_axis_num(\"t\")\\n per_t_valid = numpy.count_nonzero(~numpy.isnan(values), axis=tuple(i for i in range(values.ndim) if i != t_axis))\\n t_values = arr.coords[\"t\"].values\\n valid_t_count = int(numpy.count_nonzero(per_t_valid > 0))\\n logger.error(\\n f\"S2 diagnostics [{stage}] timesteps_with_data={valid_t_count}/{len(t_values)}, \"\\n f\"first_t={t_values[0]}, last_t={t_values[-1]}\"\\n )\\n\\n if \"bands\" in arr.dims:\\n b_axis = arr.get_axis_num(\"bands\")\\n for b_idx, b_name in enumerate(arr.coords[\"bands\"].values):\\n band_vals = numpy.take(values, indices=b_idx, axis=b_axis)\\n b_total = int(band_vals.size)\\n b_valid = int(numpy.count_nonzero(~numpy.isnan(band_vals)))\\n logger.error(\\n f\"S2 diagnostics [{stage}] band={b_name}: \"\\n f\"valid={b_valid}/{b_total} ({(100.0 * b_valid / max(b_total, 1)):.2f}%)\"\\n )\\n\\n\\ndef prepare_s2(array: xarray.DataArray, output: str, temporal_index: pandas.DatetimeIndex) -> xarray.DataArray:\\n \"\"\"\\n Prepare the Sentinel-2 data to be used as input for the CropSAR_px model.\\n :param array: input data array\\n :param output: output type\\n :param temporal_index: temporal index\\n :return: Sentinel-2 data in dimension order (t, bands, y, x)\\n \"\"\"\\n s2 = filter_s2_bands(array, output)\\n ndvi = get_ndvi(array)\\n _log_nan_stats(\"raw_selected_s2\", s2)\\n _log_nan_stats(\"raw_ndvi_for_mask\", ndvi)\\n\\n s2 = multitemporal_mask(s2, ndvi)\\n _log_nan_stats(\"after_multitemporal_mask\", s2)\\n\\n s2 = resample_s2(s2, temporal_index)\\n _log_nan_stats(\"after_resample\", s2)\\n\\n return s2.transpose(\"t\", \"bands\", \"y\", \"x\")\\n\\n\\ndef filter_s2_bands(array: xarray.DataArray, output: str) -> xarray.DataArray:\\n \"\"\"\\n Filter out the Sentinel-2 bands based on the output type.\\n :param array: input data array\\n :param output: output variable\\n :return: filtered Sentinel-2 data array\\n \"\"\"\\n return array.sel(bands=SENTINEL2_BANDS[output])\\n\\n\\ndef get_ndvi(array: xarray.DataArray) -> xarray.DataArray:\\n return array.sel(bands=SENTINEL2_NDVI)\\n\\n\\ndef resample_s2(array: xarray.DataArray, temporal_index: pandas.DatetimeIndex) -> xarray.DataArray:\\n \"\"\"\\n Resample the Sentinel-2 data to 5-day frequency, selecting the best acquisitions.\\n :param array: Sentinel-2 data\\n :param temporal_index: temporal index with 5-day frequency\\n :return: resampled Sentinel-2 data\\n \"\"\"\\n return (\\n array.resample(t=temporal_index.freqstr)\\n .map(_take_best_acquisition)\\n .reindex(t=temporal_index, method=\"ffill\", tolerance=temporal_index.freqstr)\\n )\\n\\n\\ndef _take_best_acquisition(group: xarray.DataArray) -> xarray.DataArray:\\n \"\"\"\\n Take the best acquisition from a group based on the number of not NaN pixels.\\n :param group: group of acquisitions\\n :return: best acquisition\\n \"\"\"\\n return group.isel(t=group.notnull().sum(dim=[\"bands\", \"x\", \"y\"]).argmax())\\n\\n\\ndef multitemporal_mask(s2: xarray.DataArray, ndvi: xarray.DataArray) -> xarray.DataArray:\\n \"\"\"\\n Filter out dips in the timeseries by performing multi-temporal dip detection.\\n The multi-temporal dip detection is performed on the NDVI data, the result is then applied to the Sentinel-2 data.\\n :param s2: Sentinel-2 data\\n :param ndvi: NDVI data\\n :return: masked Sentinel-2 data\\n \"\"\"\\n ndvi_mask = multitemporal_mask_ndvi(ndvi)\\n return s2.where(ndvi_mask.notnull())\\n\\n\\ndef multitemporal_mask_ndvi(ndvi: xarray.DataArray) -> xarray.DataArray:\\n \"\"\"\\n Apply multi-temporal dip detection to NDVI data.\\n :param ndvi: NDVI data\\n :return: masked NDVI data\\n \"\"\"\\n timestamps = list(ndvi.t.values)\\n daily_daterange = pandas.date_range(\\n start=timestamps[0], end=timestamps[-1] + pandas.Timedelta(days=1), freq=\"D\"\\n ).floor(\"D\")\\n ndvi_daily = ndvi.reindex(t=daily_daterange, method=\"bfill\", tolerance=\"1D\")\\n\\n # run multi-temporal dip detection\\n step = 256\\n for idx in numpy.r_[: ndvi_daily.values.shape[1] : step]:\\n for idy in numpy.r_[: ndvi_daily.values.shape[2] : step]:\\n ndvi_daily.values[:, idx : idx + step, idy : idy + step] = flaglocalminima(\\n ndvi_daily.values[:, idx : idx + step, idy : idy + step],\\n maxdip=0.01,\\n maxdif=0.1,\\n maxgap=60,\\n maxpasses=5,\\n )\\n # get the original timestamps\\n return ndvi_daily.sel(t=timestamps, method=\"ffill\", tolerance=\"1D\")\\n')\n __stickytape_write_module('cropsar_px_openeo/udf/cropsar_2d/preprocess/filter_dips.py', b'import numbers\\nfrom typing import Union\\n\\nimport numpy as np\\n\\nfrom cropsar_px_openeo.utils.logger import Logger\\n\\nlogger = Logger(__name__, udf=True)\\n\\n# based on https://github.com/WorldCereal/worldcereal-classification/blob/v1.1.1/src/worldcereal/utils/masking.py\\n\\n\\ndef flaglocalminima(\\n npdatacube: np.ndarray,\\n maxdip: Union[float, None] = None,\\n maxdif: Union[float, None] = None,\\n maxgap: Union[int, None] = None,\\n maxpasses: int = 1,\\n verbose: bool = True,\\n):\\n \"\"\"\\n Remove dips and difs (replace by np.nan) from the input npdatacube.\\n\\n dip on position i:\\n (xn - xi) < (n-l) * maxdip AND (xm - xi) < (m-i) * maxdip\\n n first not-None position with value \\'left\\' of i\\n m first not-None position with value \\'right\\' of i\\n\\n dif on position i:\\n (xn - xi) < (n-l) * maxdif OR (xm - xi) < (m-i) * maxdif\\n n first not-None position with value \\'left\\' of i\\n m first not-None position with value \\'right\\' of i\\n \"\"\"\\n return _flaglocalextrema_ct(\\n npdatacube, maxdip, maxdif, maxgap=maxgap, maxpasses=maxpasses, doflagmaxima=False, verbose=verbose\\n )\\n\\n\\ndef _flaglocalextrema_ct(\\n npdatacube: np.ndarray,\\n maxdip: Union[float, None],\\n maxdif: Union[float, None],\\n maxgap: Union[int, None] = None,\\n maxpasses: int = 1,\\n doflagmaxima: bool = False,\\n verbose: bool = True,\\n):\\n def slopeprev(npdatacube, maxgap):\\n \"\"\" \"\"\"\\n shiftedval = np.full_like(npdatacube, np.nan, dtype=float)\\n shifteddis = np.full_like(npdatacube, 1, dtype=int)\\n numberofrasters = npdatacube.shape[0]\\n shiftedval[1:numberofrasters, ...] = npdatacube[0 : numberofrasters - 1, ...]\\n\\n if np.isscalar(npdatacube[0]):\\n nans = np.isnan(npdatacube)\\n for iIdx in range(1, numberofrasters):\\n if nans[iIdx - 1]:\\n shiftedval[iIdx] = shiftedval[iIdx - 1] # can still be nan in case series started with nan\\n shifteddis[iIdx] = shifteddis[iIdx - 1] + 1\\n\\n else:\\n for iIdx in range(1, numberofrasters):\\n nans = np.isnan(npdatacube[iIdx - 1])\\n shiftedval[iIdx][nans] = shiftedval[iIdx - 1][nans]\\n shifteddis[iIdx][nans] = shifteddis[iIdx - 1][nans] + 1\\n\\n slopetoprev = (shiftedval - npdatacube) / shifteddis\\n comparable = ~np.isnan(slopetoprev)\\n if maxgap is not None:\\n comparable &= shifteddis <= maxgap\\n\\n return slopetoprev, comparable\\n\\n def slopenext(npdatacube, maxgap):\\n \"\"\" \"\"\"\\n shiftedval = np.full_like(npdatacube, np.nan, dtype=float)\\n shifteddis = np.full_like(npdatacube, 1, dtype=int)\\n numberofrasters = npdatacube.shape[0]\\n shiftedval[0 : numberofrasters - 1, ...] = npdatacube[1:numberofrasters, ...]\\n\\n if np.isscalar(npdatacube[0]):\\n nans = np.isnan(npdatacube)\\n for iIdx in range(numberofrasters - 2, -1, -1):\\n if nans[iIdx + 1]:\\n shiftedval[iIdx] = shiftedval[iIdx + 1] # can still be nan in case series started with nan\\n shifteddis[iIdx] = shifteddis[iIdx + 1] + 1\\n\\n else:\\n for iIdx in range(numberofrasters - 2, -1, -1):\\n nans = np.isnan(npdatacube[iIdx + 1])\\n shiftedval[iIdx][nans] = shiftedval[iIdx + 1][nans]\\n shifteddis[iIdx][nans] = shifteddis[iIdx + 1][nans] + 1\\n\\n slopetonext = (shiftedval - npdatacube) / shifteddis\\n comparable = ~np.isnan(slopetonext)\\n if maxgap is not None:\\n comparable &= shifteddis <= maxgap\\n\\n return slopetonext, comparable\\n\\n def masklocalminima(slopesraster, thresholdvalue):\\n return slopesraster > thresholdvalue\\n\\n def masklocalmaxima(slopesraster, thresholdvalue):\\n return slopesraster < thresholdvalue\\n\\n maskextrema = masklocalmaxima if doflagmaxima else masklocalminima\\n\\n if maxdip is not None and (not isinstance(maxdip, numbers.Real) or (float(maxdip) != maxdip) or (maxdip <= 0)):\\n raise ValueError(\"maxdip must be positive number or None\")\\n if maxdif is not None and (not isinstance(maxdif, numbers.Real) or (float(maxdif) != maxdif) or (maxdif <= 0)):\\n raise ValueError(\"maxdif must be positive number or None\")\\n if maxgap is not None and (not isinstance(maxgap, numbers.Real) or (int(maxgap) != maxgap) or (maxgap <= 0)):\\n raise ValueError(\"maxgap must be positive integer or None\")\\n\\n initialnumberofvalues = np.sum(~np.isnan(npdatacube))\\n previousnumberofvalues = initialnumberofvalues\\n for iteration in range(maxpasses):\\n prevslope, prevcomparable = slopeprev(npdatacube, maxgap)\\n nextslope, nextcomparable = slopenext(npdatacube, maxgap)\\n\\n isdip = None\\n if maxdip is not None:\\n isdip = prevcomparable & nextcomparable\\n isdip[isdip] = isdip[isdip] & maskextrema(prevslope[isdip], maxdip)\\n isdip[isdip] = isdip[isdip] & maskextrema(nextslope[isdip], maxdip)\\n\\n isdif = None\\n if maxdif is not None:\\n isdif = np.full_like(npdatacube, False, dtype=bool)\\n isdif[prevcomparable] = isdif[prevcomparable] | maskextrema(prevslope[prevcomparable], maxdif)\\n isdif[nextcomparable] = isdif[nextcomparable] | maskextrema(nextslope[nextcomparable], maxdif)\\n\\n if isdip is not None:\\n npdatacube[isdip] = np.nan\\n if isdif is not None:\\n npdatacube[isdif] = np.nan\\n\\n remainingnumberofvalues = np.sum(~np.isnan(npdatacube))\\n removednumberofvalues = previousnumberofvalues - remainingnumberofvalues\\n if verbose:\\n logger.debug(\\n \"localextrema_ct pass(%s) - removed %s values. %s values remaining. %s values removed in total\"\\n % (\\n iteration + 1,\\n removednumberofvalues,\\n remainingnumberofvalues,\\n initialnumberofvalues - remainingnumberofvalues,\\n )\\n )\\n previousnumberofvalues = remainingnumberofvalues\\n if removednumberofvalues <= 0 and maxpasses > 1:\\n if verbose:\\n logger.debug(\"localextrema_ct pass(%s) - exits\" % (iteration + 1))\\n break\\n\\n return npdatacube\\n')\n __stickytape_write_module('cropsar_px_openeo/udf/cropsar_2d/utils/models.py', b'import json\\nimport os\\nfrom pathlib import Path\\n\\nfrom cropsar_px_openeo.udf.constants import ONNX_MODEL_PATHS, SCALER_PATHS, MODEL_URLS\\nfrom cropsar_px_openeo.utils.logger import Logger\\n\\nlogger = Logger(__name__, udf=True)\\n\\n_session_cache = {}\\n_scaler_cache = {}\\n_model_cache = {}\\n\\n\\ndef load_model(output: str):\\n \"\"\"Load and cache the vito_cropsar InpaintingCnnTransformer for the given output type.\\n\\n Downloads the model zip from MODEL_URLS on first call, extracts it to\\n tmp/models/, then loads via InpaintingCnnTransformer.from_dir().\\n Used by the py8_legacy UDF at executor runtime.\\n \"\"\"\\n if output not in _model_cache:\\n from vito_cropsar.models import InpaintingCnnTransformer\\n from cropsar_px_openeo.udf.cropsar_2d.utils.helpers import download, unzip\\n\\n model_url = MODEL_URLS[output]\\n model_zip_name = os.path.basename(model_url)\\n model_name, _ = os.path.splitext(model_zip_name)\\n\\n model_base_dir = Path(\"tmp/models\")\\n model_base_dir.mkdir(parents=True, exist_ok=True)\\n model_dir = model_base_dir / model_name\\n\\n if not model_dir.exists():\\n zip_path = model_base_dir / model_zip_name\\n logger.info(f\"Downloading model {output} from {model_url}\")\\n download(model_url, zip_path)\\n logger.info(f\"Extracting {zip_path} to {model_base_dir}\")\\n unzip(zip_path, model_base_dir)\\n\\n logger.info(f\"Loading model {output} from {model_dir}\")\\n _model_cache[output] = InpaintingCnnTransformer.from_dir(str(model_dir))\\n\\n return _model_cache[output]\\n\\n\\ndef load_onnx_session(output: str):\\n \"\"\"Load and cache the ONNX InferenceSession for the given output type.\"\"\"\\n if output not in _session_cache:\\n import onnxruntime as ort\\n\\n model_path = ONNX_MODEL_PATHS[output]\\n if not os.path.exists(model_path):\\n raise FileNotFoundError(f\"ONNX model not found: {model_path}\")\\n\\n # Reduce peak memory on Spark executors:\\n # - disable BFC arena allocator (avoids large pre-allocated blocks)\\n # - disable memory pattern optimisation\\n # - limit parallelism to 1 thread\\n opts = ort.SessionOptions()\\n opts.enable_cpu_mem_arena = False\\n opts.enable_mem_pattern = False\\n opts.intra_op_num_threads = 1\\n opts.inter_op_num_threads = 1\\n\\n _session_cache[output] = ort.InferenceSession(\\n model_path, sess_options=opts, providers=[\"CPUExecutionProvider\"]\\n )\\n logger.info(f\"Loaded ONNX session for {output}\")\\n return _session_cache[output]\\n\\n\\ndef load_scaler_config(output: str) -> dict:\\n \"\"\"Load and cache the scaler configuration for the given output type.\"\"\"\\n if output not in _scaler_cache:\\n scaler_path = SCALER_PATHS[output]\\n if not os.path.exists(scaler_path):\\n raise FileNotFoundError(f\"Scaler config not found: {scaler_path}\")\\n with open(scaler_path) as f:\\n _scaler_cache[output] = json.load(f)\\n logger.info(f\"Loaded scaler config for {output}\")\\n return _scaler_cache[output]\\n')\n __stickytape_write_module('cropsar_px_openeo/udf/cropsar_2d/utils/helpers.py', b'import os\\nimport zipfile\\nfrom typing import Union\\n\\nimport requests\\nfrom cropsar_px_openeo.udf.constants import DOWNLOAD_CHUNK_SIZE, DOWNLOAD_TIMEOUT\\nfrom cropsar_px_openeo.utils.logger import Logger\\n\\nlogger = Logger(__name__, udf=True)\\n\\n\\ndef download(url: str, file_path: Union[str, os.PathLike]):\\n \"\"\"\\n Download a file from a URL and save it to the specified path.\\n :param url: URL to download\\n :param file_path: path to save the file to\\n \"\"\"\\n logger.debug(f\"Downloading {url} to {file_path}\")\\n with requests.get(url, stream=True, timeout=DOWNLOAD_TIMEOUT) as r:\\n r.raise_for_status()\\n with open(file_path, \"wb\") as f:\\n for chunk in r.iter_content(chunk_size=DOWNLOAD_CHUNK_SIZE):\\n f.write(chunk)\\n\\n\\ndef unzip(zip_path: Union[str, os.PathLike], target_path: Union[str, os.PathLike]):\\n \"\"\"\\n Unzip a ZIP-file to the target path.\\n :param zip_path: path of the ZIP-file\\n :param target_path: target path\\n \"\"\"\\n logger.debug(f\"Unzipping {zip_path} to {target_path}\")\\n with zipfile.ZipFile(zip_path) as z:\\n z.extractall(target_path)\\n')\n # ruff: noqa: E402\n \"\"\"CropSAR 2D UDF — ONNX-based inference (Python 3.11 compatible).\n \n This UDF replaces the PyTorch/vito_cropsar-based pipeline with a pure\n numpy + onnxruntime implementation. The neural network weights live in\n .onnx files; all pre/post-processing (scaling, S1 NaN interpolation,\n edge representation, temporal chunking) is done in numpy.\n \"\"\"\n from cropsar_px_openeo.udf.cropsar_2d.utils.env import setup_env\n \n setup_env()\n \n import time\n \n import numpy\n import numpy as np\n import pandas\n import xarray\n from openeo.udf import XarrayDataCube\n \n from cropsar_px_openeo.types import Output\n from cropsar_px_openeo.udf.constants import (\n MODEL_RESOLUTION,\n N_CHANNELS_OUT,\n N_TS,\n SENTINEL2_BANDS,\n TEMPORAL_BUFFER,\n TEMPORAL_FREQUENCY,\n )\n from cropsar_px_openeo.udf.cropsar_2d.preprocess.s1 import prepare_s1\n from cropsar_px_openeo.udf.cropsar_2d.preprocess.s2 import prepare_s2\n from cropsar_px_openeo.udf.cropsar_2d.utils.models import load_onnx_session, load_scaler_config\n from cropsar_px_openeo.utils.logger import Logger\n \n logger = Logger(__name__, udf=True)\n start = time.time()\n \n try:\n import cv2\n \n _HAS_CV2 = True\n except ImportError:\n _HAS_CV2 = False\n logger.warning(\"cv2 not available — edge representation disabled, quality may be reduced\")\n \n \n # ---------------------------------------------------------------------------\n # Scaler (numpy port of vito_cropsar.data.scaler.Scaler._scale)\n # ---------------------------------------------------------------------------\n def _scale_band(x, v_range, v_min=-1.0, v_max=1.0, clip=True):\n \"\"\"Scale an array from its physical range to [v_min, v_max].\"\"\"\n out = x.copy().astype(np.float32)\n out -= v_range[0]\n out /= v_range[1] - v_range[0]\n out *= v_max - v_min\n out += v_min\n if clip:\n out = np.clip(out, v_min, v_max)\n return out\n \n \n def _inverse_scale_band(x, v_range, v_min=-1.0, v_max=1.0):\n \"\"\"Inverse-scale an array from [v_min, v_max] back to its physical range.\"\"\"\n out = x.copy().astype(np.float32)\n out -= v_min\n out /= v_max - v_min\n out *= v_range[1] - v_range[0]\n out += v_range[0]\n return out\n \n \n def _apply_scaler(s1, s2, scaler_cfg):\n \"\"\"Scale s1 (T,C,H,W) and s2 (T,C,H,W) using the scaler config.\"\"\"\n v_min = scaler_cfg[\"v_min\"]\n v_max = scaler_cfg[\"v_max\"]\n ranges = scaler_cfg[\"ranges\"]\n \n s1_out = s1.copy().astype(np.float32)\n for i, band in enumerate(scaler_cfg[\"bands_s1\"]):\n s1_out[:, i] = _scale_band(s1[:, i], ranges[band], v_min, v_max)\n \n s2_out = s2.copy().astype(np.float32)\n for i, band in enumerate(scaler_cfg[\"bands_s2\"]):\n s2_out[:, i] = _scale_band(s2[:, i], ranges[band], v_min, v_max)\n \n return s1_out, s2_out\n \n \n def _inverse_scale_output(pred, scaler_cfg):\n \"\"\"Inverse-scale the model prediction (T, C_out, H, W).\"\"\"\n v_min = scaler_cfg[\"v_min\"]\n v_max = scaler_cfg[\"v_max\"]\n ranges = scaler_cfg[\"ranges\"]\n \n out = pred.copy()\n for i, band in enumerate(scaler_cfg[\"bands_s2\"]):\n out[:, i] = _inverse_scale_band(pred[:, i], ranges[band], v_min, v_max)\n return out\n \n \n # ---------------------------------------------------------------------------\n # S1 NaN interpolation (numpy port of vito_cropsar.models.shared.utils)\n # ---------------------------------------------------------------------------\n def _interpolate_s1(s1):\n \"\"\"Interpolate NaN timesteps in S1. Shape: (T, C, H, W).\n \n If ANY pixel in a timestep is NaN, the entire timestep is set to NaN\n and then linearly interpolated from neighbouring valid timesteps.\n \"\"\"\n result = s1.copy()\n for c in range(result.shape[1]):\n result[:, c] = _interpolate_s1_channel(result[:, c])\n return result\n \n \n def _interpolate_s1_channel(x):\n \"\"\"Interpolate a single S1 channel. Shape: (T, H, W).\"\"\"\n x = x.copy()\n is_nan_idx = []\n is_not_nan_idx = []\n for i in range(x.shape[0]):\n if np.isnan(x[i]).any():\n x[i] = np.nan\n is_nan_idx.append(i)\n else:\n is_not_nan_idx.append(i)\n \n if not is_not_nan_idx:\n # All timesteps have NaN — zero-fill full-NaN pixels and retry\n all_nan_mask = np.isnan(x).all(axis=0)\n x[:, all_nan_mask] = 0.0\n return _interpolate_s1_channel(x)\n \n for i in is_nan_idx:\n a, b = _get_interp_bounds(i, is_not_nan_idx)\n if a is None:\n x[i] = x[b].copy()\n elif b is None:\n x[i] = x[a].copy()\n else:\n x[i] = abs(i - b) * x[a] / (b - a) + abs(i - a) * x[b] / (b - a)\n \n return x\n \n \n def _get_interp_bounds(i, lst):\n \"\"\"Get closest lower and upper bounds in a sorted list.\"\"\"\n if i < lst[0]:\n return None, lst[0]\n if i > lst[-1]:\n return lst[-1], None\n for j in range(len(lst) - 1):\n if lst[j] <= i <= lst[j + 1]:\n return lst[j], lst[j + 1]\n raise ValueError(\"Unreachable\")\n \n \n # ---------------------------------------------------------------------------\n # Edge representation (numpy + cv2 port of vito_cropsar.models.CnnTransformer.utils)\n # ---------------------------------------------------------------------------\n def _get_edge_repr(s2):\n \"\"\"Edge representation from scaled S2 (NaN still present).\n \n Input: (T, C, H, W) → Output: (T, H, W), float32 in [0, 1].\n \"\"\"\n if not _HAS_CV2:\n return np.zeros((s2.shape[0], s2.shape[2], s2.shape[3]), dtype=np.float32)\n \n not_nan = [np.mean(~np.isnan(ts)) >= 0.2 for ts in s2]\n if not any(not_nan):\n return np.zeros((s2.shape[0], s2.shape[2], s2.shape[3]), dtype=np.float32)\n \n x_ = s2[not_nan]\n e = np.stack([_get_edges_ts(x_t) for x_t in x_]).sum(axis=0)\n d = (~np.isnan(x_)).any(axis=1).sum(axis=0)\n r = e / np.clip(d, a_min=1, a_max=None)\n vmin_q, vmax_q = np.quantile(r, (0.00, 0.99))\n r = np.clip((r - vmin_q) / max(float(vmax_q - vmin_q), 1e-6), 0, 1) ** 2\n \n return np.stack([r] * s2.shape[0]).astype(np.float32)\n \n \n def _get_edges_ts(x):\n \"\"\"Edges for one timestep averaged across channels. (C,H,W) → (H,W).\"\"\"\n edges = np.stack([_get_edges_ch(x[i]) for i in range(x.shape[0])]).mean(axis=0)\n return np.zeros_like(edges) if np.isnan(edges).any() else edges\n \n \n def _get_edges_ch(x):\n \"\"\"Edges for one channel via cv2.Canny. (H,W) → (H,W).\"\"\"\n if np.isnan(x).all():\n return np.zeros_like(x, dtype=np.float32)\n \n x_ = (x - np.nanmin(x)) / max(float(np.nanmax(x) - np.nanmin(x)), 1e-6)\n x_[np.isnan(x)] = 0.0\n \n x_ = cv2.Canny(\n image=(255 * x_).astype(np.uint8),\n threshold1=0,\n threshold2=100,\n ).astype(np.float32) / 255.0\n \n is_nan_dilated = cv2.dilate(\n np.isnan(x).astype(np.uint8),\n kernel=np.ones((3, 3), np.uint8),\n iterations=1,\n ).astype(bool)\n x_[is_nan_dilated] = 0.0\n return x_\n \n \n # ---------------------------------------------------------------------------\n # Single-chunk ONNX prediction\n # ---------------------------------------------------------------------------\n def _predict_chunk(s2_chunk, s1_chunk, session, scaler_cfg):\n \"\"\"Run one ONNX inference on a pre-sliced chunk.\n \n Parameters\n ----------\n s2_chunk : (T, C_s2, H, W) float32, raw (unscaled)\n s1_chunk : (T, C_s1, H, W) float32, raw (unscaled)\n session : onnxruntime.InferenceSession\n scaler_cfg : dict\n \n Returns\n -------\n (T, C_out, H, W) float32, inverse-scaled prediction\n \"\"\"\n # 1. Scale to [-1, 1]\n s1_s, s2_s = _apply_scaler(s1_chunk, s2_chunk, scaler_cfg)\n \n # 2. Mask: propagate NaN across channels (if ALL channels NaN → NaN)\n mask = (~np.isnan(s2_s)).any(axis=1).astype(np.int8) # (T, H, W)\n mask_bc = np.repeat(mask[:, np.newaxis], s2_s.shape[1], axis=1)\n s2_s[mask_bc == 0] = np.nan\n \n # 3. Interpolate S1 NaN timesteps (linear between valid timesteps)\n s1_s = _interpolate_s1(s1_s)\n \n # 4. Edge representation (from scaled S2, before NaN fill)\n e = _get_edge_repr(s2_s) # (T, H, W)\n \n # 5. Add batch dim → (1, T, C, H, W) / (1, T, H, W)\n s2_in = s2_s[np.newaxis].astype(np.float32)\n s1_in = s1_s[np.newaxis].astype(np.float32)\n e_in = e[np.newaxis].astype(np.float32)\n \n # 6. ONNX inference (model does nan_to_num internally)\n result = session.run(None, {\"s2\": s2_in, \"s1\": s1_in, \"e\": e_in})[0]\n result = result[0] # drop batch → (T, C_out, H, W)\n \n # 7. Inverse-scale back to physical units\n return _inverse_scale_output(result, scaler_cfg)\n \n \n # ---------------------------------------------------------------------------\n # Predict arbitrary shape (temporal + spatial chunking with weighted stitching)\n # Port of vito_cropsar.inference.predict_arbitrary_shape\n # ---------------------------------------------------------------------------\n def _compose_weight(overlap_low, overlap_high, res_model):\n \"\"\"Compose a weight vector for soft blending in overlap regions.\"\"\"\n overlap_low_trunc = min(overlap_low, int(res_model // 2))\n overlap_high_trunc = min(overlap_high, int(res_model // 2))\n weight = np.concatenate((\n ((np.arange(overlap_low) + 0.5) / overlap_low)[:overlap_low_trunc],\n np.ones(res_model - overlap_low_trunc - overlap_high_trunc),\n (1 - (np.arange(overlap_high) + 0.5) / overlap_high)[-overlap_high_trunc:],\n )).astype(np.float32)\n return weight\n \n \n def _predict_arbitrary_shape(\n s2,\n s1,\n session,\n scaler_cfg,\n n_ts=N_TS,\n resolution=MODEL_RESOLUTION,\n n_channels_out=N_CHANNELS_OUT,\n t_overlap=4,\n xy_overlap=20,\n ):\n \"\"\"Predict on an arbitrarily-sized input stack by chunking in time and\n space with weighted overlap stitching.\n \n Parameters\n ----------\n s2 : (T, C_s2, H, W) raw (unscaled)\n s1 : (T, C_s1, H, W) raw (unscaled)\n session : onnxruntime.InferenceSession\n scaler_cfg : dict\n \"\"\"\n t_res_source, _, x_res_source, y_res_source = s2.shape\n \n assert n_ts <= t_res_source, (\n f\"Temporal dimension ({t_res_source}) must be >= model n_ts ({n_ts})\"\n )\n assert resolution <= x_res_source, (\n f\"Spatial X ({x_res_source}) must be >= model resolution ({resolution})\"\n )\n assert resolution <= y_res_source, (\n f\"Spatial Y ({y_res_source}) must be >= model resolution ({resolution})\"\n )\n \n # --- Chunk layout ---\n t_chunk_count = int(np.ceil((t_res_source - t_overlap) / (n_ts - t_overlap)))\n x_chunk_count = int(np.ceil((x_res_source - xy_overlap) / (resolution - xy_overlap)))\n y_chunk_count = int(np.ceil((y_res_source - xy_overlap) / (resolution - xy_overlap)))\n \n t_overlap_avg = (\n ((t_chunk_count * n_ts) - t_res_source) / (t_chunk_count - 1)\n if t_chunk_count > 1 else 0\n )\n x_overlap_avg = (\n ((x_chunk_count * resolution) - x_res_source) / (x_chunk_count - 1)\n if x_chunk_count > 1 else 0\n )\n y_overlap_avg = (\n ((y_chunk_count * resolution) - y_res_source) / (y_chunk_count - 1)\n if y_chunk_count > 1 else 0\n )\n \n t_starts = np.floor(np.arange(t_chunk_count) * (n_ts - t_overlap_avg)).astype(np.int16)\n x_starts = np.floor(np.arange(x_chunk_count) * (resolution - x_overlap_avg)).astype(np.int16)\n y_starts = np.floor(np.arange(y_chunk_count) * (resolution - y_overlap_avg)).astype(np.int16)\n \n t_overlaps = np.pad(n_ts - np.diff(t_starts), (1, 1))\n x_overlaps = np.pad(resolution - np.diff(x_starts), (1, 1))\n y_overlaps = np.pad(resolution - np.diff(y_starts), (1, 1))\n \n output = np.zeros(\n (t_res_source, n_channels_out, x_res_source, y_res_source), dtype=np.float32\n )\n output_weights = np.zeros_like(output)\n \n for t_start, t_ov_lo, t_ov_hi in zip(t_starts, t_overlaps[:-1], t_overlaps[1:]):\n t_weight = _compose_weight(t_ov_lo, t_ov_hi, n_ts)\n \n for x_start, x_ov_lo, x_ov_hi in zip(x_starts, x_overlaps[:-1], x_overlaps[1:]):\n x_weight = _compose_weight(x_ov_lo, x_ov_hi, resolution)\n \n for y_start, y_ov_lo, y_ov_hi in zip(y_starts, y_overlaps[:-1], y_overlaps[1:]):\n y_weight = _compose_weight(y_ov_lo, y_ov_hi, resolution)\n \n s2_c = s2[\n t_start:t_start + n_ts, :,\n x_start:x_start + resolution,\n y_start:y_start + resolution,\n ]\n s1_c = s1[\n t_start:t_start + n_ts, :,\n x_start:x_start + resolution,\n y_start:y_start + resolution,\n ]\n \n pred = _predict_chunk(s2_c, s1_c, session, scaler_cfg)\n \n wt = np.broadcast_to(t_weight.reshape(-1, 1, 1, 1), pred.shape)\n wx = np.broadcast_to(x_weight.reshape(1, 1, -1, 1), pred.shape)\n wy = np.broadcast_to(y_weight.reshape(1, 1, 1, -1), pred.shape)\n w = np.minimum(wx, wy) * wt\n \n sl = np.s_[\n t_start:t_start + n_ts, :,\n x_start:x_start + resolution,\n y_start:y_start + resolution,\n ]\n output[sl] += pred * w\n output_weights[sl] += w\n \n return output / output_weights\n \n \n def _nan_valid_counts(data: np.ndarray) -> tuple[int, int]:\n \"\"\"Return valid and total element counts for an ndarray.\"\"\"\n total = int(data.size)\n valid = int(np.count_nonzero(~np.isnan(data)))\n return valid, total\n \n \n def _log_nan_diagnostics(label: str, data_array: xarray.DataArray) -> None:\n \"\"\"Log detailed NaN diagnostics for a DataArray at error level.\"\"\"\n values = data_array.values\n valid, total = _nan_valid_counts(values)\n logger.info(\n f\"NaN diagnostics [{label}] shape={values.shape}, dims={data_array.dims}, \"\n f\"valid={valid}/{total} ({(100.0 * valid / max(total, 1)):.2f}%)\"\n )\n \n if \"bands\" in data_array.dims:\n band_dim = data_array.get_axis_num(\"bands\")\n for band_idx, band_name in enumerate(data_array.coords[\"bands\"].values):\n band_values = np.take(values, indices=band_idx, axis=band_dim)\n b_valid, b_total = _nan_valid_counts(band_values)\n logger.info(\n f\"NaN diagnostics [{label}] band={band_name}: \"\n f\"valid={b_valid}/{b_total} ({(100.0 * b_valid / max(b_total, 1)):.2f}%)\"\n )\n \n if \"t\" in data_array.dims:\n t_dim = data_array.get_axis_num(\"t\")\n per_t_valid = np.count_nonzero(~np.isnan(values), axis=tuple(i for i in range(values.ndim) if i != t_dim))\n worst_t_idx = int(np.argmin(per_t_valid))\n best_t_idx = int(np.argmax(per_t_valid))\n t_values = data_array.coords[\"t\"].values\n logger.info(\n f\"NaN diagnostics [{label}] timestep_valid_min={int(per_t_valid[worst_t_idx])} \"\n f\"at t={t_values[worst_t_idx]}, timestep_valid_max={int(per_t_valid[best_t_idx])} \"\n f\"at t={t_values[best_t_idx]}\"\n )\n \n \n \n # ---------------------------------------------------------------------------\n # openEO UDF entry points\n # ---------------------------------------------------------------------------\n def apply_datacube(cube: XarrayDataCube, context: dict) -> XarrayDataCube: # noqa\n logger.info(str(context))\n \n startdate = context[\"startdate\"]\n enddate = context[\"enddate\"]\n output = context[\"output\"]\n \n result = process(array=cube.array, startdate=startdate, enddate=enddate, output=output)\n return XarrayDataCube(result)\n \n \n def log_time(message: str, previous=time.time()) -> float:\n now = time.time()\n logger.debug(f\"{message} ({previous - time.time()} seconds)\")\n return now\n \n \n def process(\n array: xarray.DataArray,\n startdate: str,\n enddate: str,\n output: Output,\n ) -> xarray.DataArray:\n \"\"\"Apply the CropSAR 2D algorithm using ONNX inference.\n \n :param array: input data (Sentinel-1 + Sentinel-2)\n :param startdate: requested start date\n :param enddate: requested end date\n :param output: output type\n :return:\n \"\"\"\n t = log_time(\"Initiated environment\")\n \n logger.info(\n f\"Process context: startdate={startdate}, enddate={enddate}, output={output}, \"\n f\"input_shape={array.shape}, input_dims={array.dims}\"\n )\n \n input_temporal_index = pandas.date_range(\n start=pandas.to_datetime(startdate) - pandas.to_timedelta(TEMPORAL_BUFFER, unit=\"day\"),\n end=pandas.to_datetime(enddate) + pandas.to_timedelta(TEMPORAL_BUFFER, unit=\"day\"),\n freq=TEMPORAL_FREQUENCY,\n )\n output_temporal_index = pandas.date_range(\n start=pandas.to_datetime(startdate),\n end=pandas.to_datetime(enddate),\n freq=TEMPORAL_FREQUENCY,\n )\n \n s1 = prepare_s1(array, input_temporal_index)\n s2 = prepare_s2(array, output, input_temporal_index)\n t = log_time(\"Prepared data\", t)\n \n # Input checks:\n if numpy.isnan(s1).all() or numpy.isnan(s2).all():\n # don't do a prediction, because it will be based on no input data\n logger.error(\"CHECK ERROR: Not enough input data to make a prediction in this chunk, returning empty array\")\n _log_nan_diagnostics(\"raw_input\", array)\n _log_nan_diagnostics(\"prepared_s1\", s1)\n _log_nan_diagnostics(\"prepared_s2\", s2)\n \n return get_empty_array(array, output, output_temporal_index)\n \n session = load_onnx_session(output)\n scaler_cfg = load_scaler_config(output)\n t = log_time(\"Loaded model\", t)\n \n result = _predict_arbitrary_shape(\n s2=s2.values, s1=s1.values, session=session, scaler_cfg=scaler_cfg,\n )\n log_time(\"Finished predictions\", t)\n \n # Filter result to requested [startdate, enddate] range\n return xarray.DataArray(\n data=result[input_temporal_index.isin(output_temporal_index)],\n dims=[\"t\", \"bands\", \"y\", \"x\"],\n coords={\n \"bands\": SENTINEL2_BANDS[output],\n \"t\": output_temporal_index,\n \"y\": s2.y,\n \"x\": s2.x,\n },\n )\n \n \n def get_empty_array(\n array: xarray.DataArray, output: Output, temporal_index: pandas.DatetimeIndex\n ) -> xarray.DataArray:\n \"\"\"Get an empty DataArray based on the output type and the shape of the input data.\"\"\"\n output_bands = SENTINEL2_BANDS[output]\n logger.debug(\"Returning empty data array\")\n return xarray.DataArray(\n data=numpy.full(\n shape=(len(temporal_index), len(output_bands), array.y.shape[0], array.x.shape[0]),\n fill_value=numpy.nan,\n ),\n dims=[\"t\", \"bands\", \"y\", \"x\"],\n coords={\"t\": temporal_index, \"bands\": output_bands, \"y\": array.y, \"x\": array.x},\n )\n " }, "result": true } } }, "size": [ { "dimension": "x", "value": 112, "unit": "px" }, { "dimension": "y", "value": 112, "unit": "px" } ] } }, "filterspatial1": { "process_id": "filter_spatial", "arguments": { "data": { "from_node": "applyneighborhood1" }, "geometries": { "from_parameter": "geometry" } } }, "renamelabels3": { "process_id": "rename_labels", "arguments": { "data": { "from_node": "filterspatial1" }, "dimension": "bands", "target": [ { "from_parameter": "output" } ] }, "result": true } }, "id": "cropsar2d", "summary": "Cloud-free monitoring using Sentinel satellites", "description": "# CropSAR 2D\n\n## Description\n\nThe `cropsar2d` process produces Sentinel-2 data cloud-free with a regularity of five-day intervals. \nIn the current version of the service, the output types supported include:\n\n- NDVI\n- FAPAR\n- FCOVER\n\n> The 'startdate' parameter corresponds to the date of the first image in the result. \n> From this start date, a new image will be generated every five days up to, or beyond, the specified end date.\n\nBy default, the output variable is set to NDVI.\nHowever, by supplying one of the supported values listed above to the output parameter, a different result can be obtained.\n\n> Due to its reliance on particular libraries, the `cropsar2d` service can only be executed as a batch job.\n\nThe service can be used for a wide range of application domains. More information on this service can be consulted through this [blog post](https://remotesensing.vito.be//news/new-cropsar-service-provides-cloud-free-time-series-pixel-level?hs_preview=hvLlyDZj-139839072553).\n\nNote: since the correlation between optical Sentinel-2 and radar Sentinel-1 is not unique, the expected quality of the estimated optical values decreases with the length of the optical gap.\n\n## Methodology\n\nThe CropSAR 2D algorithm uses a deep learning model that combines ResNet blocks (per-image spatial encoding/decoding) and transformers (per-pixel temporal gap-filling) to generate a spatially and temporally consistent time series of the requested output. It uses collocated Sentinel-1 and Sentinel-2 datacubes as input and returns a completely cloud-free Sentinel-2-like output.", "parameters": [ { "name": "geometry", "description": "Area of interest in GeoJSON format", "schema": { "type": "object", "subtype": "geojson" } }, { "name": "startdate", "description": "Start of the temporal interval. The service will generate an image every 5 days starting from this date.", "schema": { "type": "string", "subtype": "date" } }, { "name": "enddate", "description": "End of the temporal interval.", "schema": { "type": "string", "subtype": "date" } }, { "name": "output", "description": "Output type of the result: NDVI, FAPAR, FCOVER", "schema": { "type": "string", "enum": [ "NDVI", "FAPAR", "FCOVER" ] }, "default": "NDVI", "optional": true } ] }