Skip to content

base

logger = logging.getLogger(__name__) module-attribute

Builder dataclass

Bases: ABC

Base class for Builder objects.

Provides methods to set specific values for the builder and enforce checks such that the builder can be constructed correctly and to avoid possible errors when the builder is executed.

Source code in src/recnexteval/evaluators/builder/base.py
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
@dataclass
class Builder(ABC):
    """Base class for Builder objects.

    Provides methods to set specific values for the builder and enforce checks
    such that the builder can be constructed correctly and to avoid possible
    errors when the builder is executed.
    """

    setting: Setting = field(init=False)
    """Setting to evaluate the algorithms on"""
    metric_k: int = 10
    """K value for metrics"""
    ignore_unknown_user: bool = True
    """Ignore unknown user in the evaluation"""
    ignore_unknown_item: bool = True
    """Ignore unknown item in the evaluation"""
    seed: int = 42
    """Random seed for reproducibility"""
    metric_entries: dict[str, MetricEntry] = field(default_factory=dict)
    """dict of metrics to evaluate algorithm on.
    Using dict instead of list for fast lookup"""

    def _check_setting_exist(self) -> bool:
        """Check if setting is already set.

        Returns:
            True if setting is set, False otherwise.
        """
        return hasattr(self, "setting") and self.setting is not None

    def set_metric_k(self, K: int) -> None:
        """Set K value for all metrics.

        Args:
            K: K value to set for all metrics.
        """
        self.metric_k = K

    def add_metric(self, metric: str | type) -> None:
        """Add metric to evaluate algorithm on.

        Metric will be added to the metric_entries dict where it will later be
        converted to a list when the evaluator is constructed.

        Note:
            If K is not yet specified, the setting's top_K value will be used. This
            requires the setting to be set before adding the metric.

        Args:
            metric: Metric to evaluate algorithm on.

        Raises:
            ValueError: If metric is not found in METRIC_REGISTRY.
            RuntimeError: If setting is not set.
        """
        if not self._check_setting_exist():
            raise RuntimeError(
                "Setting has not been set. To ensure conformity, of the addition of"
                " other components please set the setting first. Call add_setting() method."
            )

        metric = arg_to_str(metric)

        if metric not in METRIC_REGISTRY:
            raise ValueError(f"Metric {metric} could not be resolved.")

        if not hasattr(self, "metric_k"):
            self.metric_k = self.setting.top_K
            warn(
                "K value not yet specified before setting metric, using setting's top_K value."
                " We recommend specifying K value for metric. If you want to change the K value,"
                " you can clear all metric entry and set the K value before adding metrics."
            )

        metric_name = f"{metric}_{self.metric_k}"
        if metric_name in self.metric_entries:
            logger.warning(f"Metric {metric_name} already exists. Skipping adding metric.")
            return

        self.metric_entries[metric_name] = MetricEntry(metric, self.metric_k)

    def add_setting(self, setting: Setting) -> None:
        """Add setting to the evaluator builder.

        Note:
            The setting should be set before adding metrics or algorithms
            to the evaluator.

        Args:
            setting: Setting to evaluate the algorithms on.

        Raises:
            ValueError: If setting is not of instance Setting.
        """
        if not isinstance(setting, Setting):
            raise ValueError(f"setting should be of type Setting, got {type(setting)}")
        if hasattr(self, "setting") and self.setting is not None:
            warn("Setting is already set. Continuing will overwrite the setting.")

        self.setting = setting

    def clear_metrics(self) -> None:
        """Clear all metrics from the builder."""
        self.metric_entries.clear()

    def _check_ready(self) -> None:
        """Check if the builder is ready to construct Evaluator.

        Raises:
            RuntimeError: If there are invalid configurations.
        """
        if not hasattr(self, "metric_k"):
            self.metric_k = self.setting.top_K
            warn(
                "K value not yet specified before setting metric, using setting's top_K value."
                " We recommend specifying K value for metric. If you want to change the K value,"
                " you can clear all metric entry and set the K value before adding metrics."
            )

        if len(self.metric_entries) == 0:
            raise RuntimeError("No metrics specified, can't construct Evaluator")

        # Check for settings #
        if self.setting is None:
            raise RuntimeError("No settings specified, can't construct Evaluator")
        if not self.setting.is_ready:
            raise RuntimeError(
                "Setting is not ready, can't construct Evaluator. "
                "Call split() on the setting first."
            )

    @abstractmethod
    def build(self) -> EvaluatorBase:
        """Build object.

        Raises:
            NotImplementedError: If the method is not implemented.
        """
        raise NotImplementedError

setting = field(init=False) class-attribute instance-attribute

Setting to evaluate the algorithms on

metric_k = 10 class-attribute instance-attribute

K value for metrics

ignore_unknown_user = True class-attribute instance-attribute

Ignore unknown user in the evaluation

ignore_unknown_item = True class-attribute instance-attribute

Ignore unknown item in the evaluation

seed = 42 class-attribute instance-attribute

Random seed for reproducibility

metric_entries = field(default_factory=dict) class-attribute instance-attribute

dict of metrics to evaluate algorithm on. Using dict instead of list for fast lookup

set_metric_k(K)

Set K value for all metrics.

Parameters:

Name Type Description Default
K int

K value to set for all metrics.

required
Source code in src/recnexteval/evaluators/builder/base.py
49
50
51
52
53
54
55
def set_metric_k(self, K: int) -> None:
    """Set K value for all metrics.

    Args:
        K: K value to set for all metrics.
    """
    self.metric_k = K

add_metric(metric)

Add metric to evaluate algorithm on.

Metric will be added to the metric_entries dict where it will later be converted to a list when the evaluator is constructed.

Note

If K is not yet specified, the setting's top_K value will be used. This requires the setting to be set before adding the metric.

Parameters:

Name Type Description Default
metric str | type

Metric to evaluate algorithm on.

required

Raises:

Type Description
ValueError

If metric is not found in METRIC_REGISTRY.

RuntimeError

If setting is not set.

Source code in src/recnexteval/evaluators/builder/base.py
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
def add_metric(self, metric: str | type) -> None:
    """Add metric to evaluate algorithm on.

    Metric will be added to the metric_entries dict where it will later be
    converted to a list when the evaluator is constructed.

    Note:
        If K is not yet specified, the setting's top_K value will be used. This
        requires the setting to be set before adding the metric.

    Args:
        metric: Metric to evaluate algorithm on.

    Raises:
        ValueError: If metric is not found in METRIC_REGISTRY.
        RuntimeError: If setting is not set.
    """
    if not self._check_setting_exist():
        raise RuntimeError(
            "Setting has not been set. To ensure conformity, of the addition of"
            " other components please set the setting first. Call add_setting() method."
        )

    metric = arg_to_str(metric)

    if metric not in METRIC_REGISTRY:
        raise ValueError(f"Metric {metric} could not be resolved.")

    if not hasattr(self, "metric_k"):
        self.metric_k = self.setting.top_K
        warn(
            "K value not yet specified before setting metric, using setting's top_K value."
            " We recommend specifying K value for metric. If you want to change the K value,"
            " you can clear all metric entry and set the K value before adding metrics."
        )

    metric_name = f"{metric}_{self.metric_k}"
    if metric_name in self.metric_entries:
        logger.warning(f"Metric {metric_name} already exists. Skipping adding metric.")
        return

    self.metric_entries[metric_name] = MetricEntry(metric, self.metric_k)

add_setting(setting)

Add setting to the evaluator builder.

Note

The setting should be set before adding metrics or algorithms to the evaluator.

Parameters:

Name Type Description Default
setting Setting

Setting to evaluate the algorithms on.

required

Raises:

Type Description
ValueError

If setting is not of instance Setting.

Source code in src/recnexteval/evaluators/builder/base.py
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
def add_setting(self, setting: Setting) -> None:
    """Add setting to the evaluator builder.

    Note:
        The setting should be set before adding metrics or algorithms
        to the evaluator.

    Args:
        setting: Setting to evaluate the algorithms on.

    Raises:
        ValueError: If setting is not of instance Setting.
    """
    if not isinstance(setting, Setting):
        raise ValueError(f"setting should be of type Setting, got {type(setting)}")
    if hasattr(self, "setting") and self.setting is not None:
        warn("Setting is already set. Continuing will overwrite the setting.")

    self.setting = setting

clear_metrics()

Clear all metrics from the builder.

Source code in src/recnexteval/evaluators/builder/base.py
120
121
122
def clear_metrics(self) -> None:
    """Clear all metrics from the builder."""
    self.metric_entries.clear()

build() abstractmethod

Build object.

Raises:

Type Description
NotImplementedError

If the method is not implemented.

Source code in src/recnexteval/evaluators/builder/base.py
150
151
152
153
154
155
156
157
@abstractmethod
def build(self) -> EvaluatorBase:
    """Build object.

    Raises:
        NotImplementedError: If the method is not implemented.
    """
    raise NotImplementedError