Source code for xscen.indicators

"""Functions to compute xclim indicators."""

import logging
import os
from collections.abc import Callable, Sequence
from functools import partial
from inspect import signature
from types import ModuleType

import pandas as pd
import xarray as xr
import xclim as xc
from boltons.funcutils import wraps
from intake_esm import DerivedVariableRegistry
from xclim.core.calendar import construct_offset, parse_offset
from xclim.core.formatting import gen_call_string, update_history
from xclim.core.indicator import Indicator

from xscen.config import parse_config

from .catutils import parse_from_ds
from .utils import CV, rechunk_for_resample, standardize_periods


logger = logging.getLogger(__name__)

__all__ = ["compute_indicators", "registry_from_collection"]


def get_indicator_outputs(ind: xc.core.indicator.Indicator, in_freq: str) -> tuple[list[str], str]:
    """
    Return the variables names and resampling frequency of a given indicator.

    CAUTION: Some indicators will build the variable name on-the-fly according to the arguments.
    This function will return the template string (with "{}").

    Parameters
    ----------
    ind : Indicator
        An xclim indicator.
    in_freq : str
        The data's sampling frequency.

    Returns
    -------
    var_names : list
        List of variable names.
    freq : str
        Indicator resampling frequency. "fx" for time-reducing indicator.
    """
    if isinstance(ind, xc.core.indicator.ReducingIndicator):
        frq = "fx"
    elif not isinstance(ind, xc.core.indicator.ResamplingIndicator):
        frq = in_freq
    else:
        frq = ind.injected_parameters["freq"] if "freq" in ind.injected_parameters else ind.parameters["freq"].default
    if frq == "YS":
        frq = "YS-JAN"
    var_names = [cfa.var_name for cfa in ind.outputs]
    return var_names, frq


def _update_indicators_history(func: Callable) -> Callable:
    """
    Decorator that auto-generates and fills the history attribute.

    Fills the history attributes of the datasets in each entry of the dictionary returned by the decorated function.
    The history is generated from the signature of the function and added to the first output.
    Because of a limitation of the `boltons` wrapper, all arguments passed to the wrapped function
    will be printed as keyword arguments.

    Parameters
    ----------
    func : Callable
        The function to decorate.

    Returns
    -------
    Callable
        The decorated function.
    """

    @wraps(func)
    def _call_and_add_history(*args, **kwargs):
        """Call the function and then generate and add the history attr."""
        outs = func(*args, **kwargs)

        da_list = [arg for arg in args if isinstance(arg, xr.DataArray)]
        da_dict = {name: arg for name, arg in kwargs.items() if isinstance(arg, xr.DataArray)}

        # The wrapper hides how the user passed the arguments (positional or keyword)
        # Instead of having it all position, we have it all keyword-like for explicitness.
        bound_args = signature(func).bind(*args, **kwargs)
        call_string = gen_call_string(func.__name__, **bound_args.arguments)

        for key, val in outs.items():
            if isinstance(val, xr.Dataset):
                attr = update_history(
                    call_string,
                    *da_list,
                    **da_dict,
                )
                outs[key].attrs["history"] = attr

        return outs

    return _call_and_add_history


[docs] @_update_indicators_history @parse_config def compute_indicators( # noqa: C901 ds: xr.Dataset, indicators: (str | os.PathLike | Sequence[Indicator] | Sequence[tuple[str, Indicator]] | ModuleType), *, periods: list[str] | list[list[str]] | None = None, restrict_years: bool = True, to_level: str | None = "indicators", rechunk_input: bool = False, ) -> dict: """ Calculate variables and indicators based on a YAML call to xclim. The function cuts the output to be the same years as the inputs. Hence, if an indicator creates a timestep outside the original year range (e.g. the first DJF for QS-DEC), it will not appear in the output. Parameters ---------- ds : xr.Dataset Dataset to use for the indicators. indicators : str or os.PathLike or Sequence[Indicator] or Sequence[tuple[str, Indicator]] or ModuleType Path to a YAML file that instructs on how to calculate missing variables. Can also be only the "stem", if translations and custom indices are implemented. Can be the indicator module directly, or a sequence of indicators or a sequence of tuples (indicator name, indicator) as returned by `iter_indicators()`. periods : list of str or list of lists of str, optional Either [start, end] or list of [start, end] of continuous periods over which to compute the indicators. This is needed when the time axis of ds contains some jumps in time. If None, the dataset will be considered continuous. restrict_years : bool If True, cut the time axis to be within the same years as the input. This is mostly useful for frequencies that do not start in January, such as QS-DEC. In that instance, `xclim` would start on previous_year-12-01 (DJF), with a NaN. `restrict_years` will cut that first timestep. This should have no effect on YS and MS indicators. to_level : str, optional The processing level to assign to the output. If None, the processing level of the inputs is preserved. rechunk_input : bool If True, the dataset is rechunked with :py:func:`flox.xarray.rechunk_for_blockwise` according to the resampling frequency of the indicator. Each rechunking is done once per frequency with :py:func:`xscen.utils.rechunk_for_resample`. Returns ------- dict Dictionary (keys = timedeltas) with indicators separated by temporal resolution. See Also -------- xclim.indicators : Indicators module of xclim. xclim.IndicatorCollection.from_yaml : YAML indicator constructor function of xclim. """ if isinstance(indicators, str | os.PathLike): logger.debug("Loading indicator module.") module = xc.IndicatorCollection.from_yaml(indicators) indicators = module.iter_indicators() elif hasattr(indicators, "iter_indicators"): indicators = indicators.iter_indicators() try: N = len(indicators) except TypeError: N = None else: msg = f"Computing {N} indicators." logger.info(msg) periods = standardize_periods(periods) in_freq = xr.infer_freq(ds.time) if "time" in ds.dims else "fx" dss_rechunked = {} out_dict = {} for i, ind in enumerate(indicators, 1): if isinstance(ind, tuple): iden, ind = ind else: iden = ind.identifier msg = f"{i} - Computing {iden}." logger.info(msg) _, freq = get_indicator_outputs(ind, in_freq) if rechunk_input and freq not in ["fx", in_freq]: if freq not in dss_rechunked: msg = f"Rechunking with flox for freq {freq}." logger.debug(msg) dss_rechunked[freq] = rechunk_for_resample(ds, time=freq) else: msg = f"Using rechunked for freq {freq}" logger.debug(msg) ds_in = dss_rechunked[freq] else: ds_in = ds if periods is None: # Pandas as no semiannual frequency and 2Q is capricious if freq.startswith("2Q"): logger.debug("Dropping start of timeseries to ensure semiannual frequency works.") ds_in = fix_semiannual(ds_in, freq) # Make the call to xclim out = ind(ds=ds_in) else: # Multiple time periods to concatenate concats = [] for period in periods: # Make the call to xclim ds_subset = ds_in.sel(time=slice(period[0], period[1])) # Pandas as no semiannual frequency and 2Q is capricious if freq.startswith("2Q"): logger.debug("Dropping start of timeseries to ensure semiannual frequency works.") ds_subset = fix_semiannual(ds_subset, freq) tmp = ind(ds=ds_subset) # In order to concatenate time periods, the indicator still needs a time dimension if freq == "fx": tmp = tmp.assign_coords({"time": ds_subset.time[0]}) concats.append(tmp) out = xr.concat(concats, dim="time") # Make sure that attributes have been kept for the dimensions and coordinates. Fixes a bug in xarray. for c in set(list(out.coords) + list(out.dims)).intersection(set(list(ds.coords) + list(ds.dims))): if (out[c].attrs != ds[c].attrs) and (out[c].sizes == ds[c].sizes): out[c].attrs = ds[c].attrs if restrict_years and "time" in out.dims: # cut the time axis to be within the same years as the input # for QS-DEC, xclim starts on DJF with time previous_year-12-01 with a nan as values. We want to cut this. # this should have no effect on YS and MS indicators out = out.sel(time=slice(str(ds.time[0].dt.year.values), str(ds.time[-1].dt.year.values))) # Create the dictionary key key = freq if key not in out_dict: out_dict[key] = out out_dict[key].attrs = ds.attrs out_dict[key].attrs["cat:xrfreq"] = freq out_dict[key].attrs["cat:frequency"] = CV.xrfreq_to_frequency(freq, None) if to_level is not None: out_dict[key].attrs["cat:processing_level"] = to_level else: for v in out.data_vars: out_dict[key][v] = out[v] for key in out_dict: out_dict[key].attrs["cat:variable"] = parse_from_ds(out_dict[key], ["variable"])["variable"] return out_dict
[docs] def registry_from_collection( collection: xc.IndicatorCollection, registry: DerivedVariableRegistry | None = None, variable_column: str = "variable", ) -> DerivedVariableRegistry: """ Convert a xclim virtual indicators collection to an intake_esm Derived Variable Registry. Parameters ---------- collection : xc.IndicatorCollection A collection of xclim. registry : DerivedVariableRegistry, optional If given, this registry is extended, instead of creating a new one. variable_column : str The name of the variable column (the name used in the query). Returns ------- DerivedVariableRegistry A variable registry where each indicator and each of its output has been registered. If an indicator returns multiple values, each of them is mapped individually, as the DerivedVariableRegistry only supports single output function. Each indicator was wrapped into a new function that only accepts a dataset and returns it with the extra variable appended. This means all other parameters are given their defaults. """ dvr = registry or DerivedVariableRegistry() for _name, ind in collection.iter_indicators(): query = {variable_column: [p.default for p in ind.parameters.values() if p.kind == 0]} for output in ind.outputs: dvr.register(variable=output.var_name, query=query)(_derived_func(ind, output.var_name)) return dvr
def _ensure_list(x): if not isinstance(x, list | tuple): return [x] return x def _derived_func(ind: xc.core.indicator.Indicator, var_name: str) -> partial: def func(ds, *, ind, var_name): out = ind(ds=ds) return ds.assign({var_name: out[var_name]}).assign_attrs(history=out.attrs["history"]) func.__name__ = ind.identifier return partial(func, ind=ind, var_name=var_name) def select_inds_for_avail_vars( ds: xr.Dataset, indicators: (str | os.PathLike | Sequence[Indicator] | Sequence[tuple[str, Indicator]] | xc.IndicatorCollection | dict), ) -> xc.IndicatorCollection: """ Filter the indicators for which the necessary variables are available. Parameters ---------- ds : xr.Dataset Dataset to use for the indicators. indicators : str or os.PathLike or Sequence[Indicator] or Sequence[Tuple[str, Indicator]] Path to a YAML file that instructs on how to calculate indicators. Can also be only the "stem", if translations and custom indices are implemented. Can be the indicator module directly, or a sequence of indicators or a sequence of tuples (indicator name, indicator) as returned by `iter_indicators()`. Returns ------- xclim.IndicatorCollection An indicator module of 'length' ∈ [0, n]. See Also -------- xclim.indicators : Indicators module of xclim. xclim.IndicatorCollection.from_yaml : YAML indicator constructor function of xclim. """ # Transform the 'indicators' input into a list of tuples (name, indicator) is_list_of_tuples = isinstance(indicators, list) and all(isinstance(i, tuple) for i in indicators) if isinstance(indicators, str | os.PathLike): logger.debug("Loading indicator module.") indicators = xc.IndicatorCollection.from_yaml(indicators) if hasattr(indicators, "iter_indicators"): indicators = [(name, ind) for name, ind in indicators.iter_indicators()] elif isinstance(indicators, dict): indicators = [(name, ind) for name, ind in indicators.items()] elif isinstance(indicators, list | tuple) and not is_list_of_tuples: indicators = [(ind.identifier, ind) for ind in indicators] XCVARS = xc.core.VARIABLES available_vars = {var for var in ds.data_vars if var in XCVARS.keys()} available_inds = {name: ind for var in available_vars for name, ind in indicators if var in ind.parameters.keys()} return xc.IndicatorCollection(available_inds, name="inds_for_avail_vars") def _wrap_month(m): # Ensure the month number is between 1 and 12 # Modulo returns 0 if m is a multiple of 12, 0 is false and we want 12. return (m % 12) or 12 def fix_semiannual(ds: xr.Dataset, freq: str) -> xr.Dataset: """ Avoid wrong start dates for semiannual frequency. Resampling with offsets that are multiples of a base frequency (ex: 2QS-OCT) is broken in pandas (https://github.com/pandas-dev/pandas/issues/51563). This will cut the beginning of the dataset so it starts exactly at the beginning of the resampling period. Parameters ---------- ds : xr.Dataset The Dataset to be adjusted. freq : str Frequency string. Returns ------- xr.Dataset The adjusted Dataset. Raises ------ NotImplementedError When a non-2Q frequency is passed. ValueError If a start date cannot be found in dataset. """ # I hate that we have to do that mul, b, s, anc = parse_offset(freq) if mul != 2 or b != "Q": raise NotImplementedError("This only fixes 2Q frequencies.") # Get MONTH: N mapping (invert xarray's) months_inv = xr.coding.cftime_offsets._MONTH_ABBREVIATIONS months = dict(zip(months_inv.values(), months_inv.keys(), strict=False)) if s: m1 = months[anc] else: m1 = _wrap_month(months[anc] + 1) freq = construct_offset(mul, b, True, months_inv[m1]) m2 = _wrap_month(m1 + 6) time = ds.indexes["time"] if isinstance(time, xr.CFTimeIndex): offset = xr.coding.cftime_offsets.to_offset(freq) is_on_offset = offset.onOffset else: offset = pd.tseries.frequencies.to_offset(freq) is_on_offset = offset.is_on_offset if is_on_offset(time[0]) and time[0].month in (m1, m2): # wow, already correct! return ds for t in time: if is_on_offset(t) and t.month in (m1, m2): return ds.sel(time=(time >= t)) raise ValueError(f"Can't find a start date that fits with frequency {freq}.")