Skip to content

core

MetricAccumulator

Source code in src/recnexteval/evaluators/core/accumulator.py
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
class MetricAccumulator:
    def __init__(self) -> None:
        self.acc: defaultdict[str, dict[str, Metric]] = defaultdict(dict)

    def __getitem__(self, key) -> dict[str, Metric]:
        return self.acc[key]

    def add(self, metric: Metric, algorithm_name: str) -> None:
        """Add a metric to the accumulator.

        Takes a Metric object and adds it under the algorithm name. If
        the specified metric already exists for the algorithm, it will be
        overwritten with the new metric.

        Args:
            metric: Metric to store.
            algorithm_name: Name of the algorithm.
        """
        if metric.identifier in self.acc[algorithm_name]:
            logger.warning(
                f"Metric {metric.identifier} already exists for algorithm {algorithm_name}. Overwriting..."
            )

        logger.debug(f"Metric {metric.identifier} created for algorithm {algorithm_name}")

        self.acc[algorithm_name][metric.identifier] = metric

    @property
    def user_level_metrics(self) -> defaultdict:
        results = defaultdict()
        for algo_name in self.acc:
            for metric_identifier in self.acc[algo_name]:
                metric = self.acc[algo_name][metric_identifier]
                results[(algo_name, metric.timestamp_limit, metric.name)] = (
                    metric.micro_result
                )
        return results

    @property
    def window_level_metrics(self) -> defaultdict:
        results = defaultdict(dict)
        for algo_name in self.acc:
            for metric_identifier in self.acc[algo_name]:
                metric = self.acc[algo_name][metric_identifier]
                score = metric.macro_result
                num_user = metric.num_users
                if score == 0 and num_user == 0:
                    logger.info(
                        f"Metric {metric.name} for algorithm {algo_name} "
                        f"at t={metric.timestamp_limit} has 0 score and 0 users. "
                        "The ground truth may be empty due to no interactions occurring in that window."
                    )
                elif score == 0 and num_user != 0:
                    logger.info(
                        f"Metric {metric.name} for algorithm {algo_name} "
                        f"at t={metric.timestamp_limit} has 0 score but there are interactions. "
                        f"{algo_name} did not have any correct predictions."
                    )
                results[(algo_name, metric.timestamp_limit, metric.name)]["score"] = score
                results[(algo_name, metric.timestamp_limit, metric.name)]["num_user"] = (
                    num_user
                )
        return results

    def df_user_level_metric(self) -> pd.DataFrame:
        """Get user-level metrics across all timestamps.

        Returns:
            DataFrame with user-level metric computations.
        """
        df = pd.DataFrame.from_dict(self.user_level_metrics, orient="index").explode(
            ["user_id", "score"]
        )
        df = df.rename_axis(["algorithm", "timestamp", "metric"])
        df.rename(columns={"score": "user_score"}, inplace=True)
        return df

    def df_window_level_metric(self) -> pd.DataFrame:
        df = pd.DataFrame.from_dict(self.window_level_metrics, orient="index").explode(
            ["score", "num_user"]
        )
        df = df.rename_axis(["algorithm", "timestamp", "metric"])
        df.rename(columns={"score": "window_score"}, inplace=True)
        return df

    def df_macro_level_metric(self) -> pd.DataFrame:
        """Get macro-level metrics across all timestamps.

        Returns:
            DataFrame with macro-level metric computations.
        """
        df = pd.DataFrame.from_dict(self.window_level_metrics, orient="index").explode(
            ["score", "num_user"]
        )
        df = df.rename_axis(["algorithm", "timestamp", "metric"])
        result = df.groupby(["algorithm", "metric"]).mean()["score"].to_frame()
        result["num_window"] = df.groupby(["algorithm", "metric"]).count()["score"]
        result = result.rename(columns={"score": "macro_score"})
        return result

    def df_micro_level_metric(self) -> pd.DataFrame:
        """Get micro-level metrics across all timestamps.

        Returns:
            DataFrame with micro-level metric computations.
        """
        df = pd.DataFrame.from_dict(self.user_level_metrics, orient="index").explode(
            ["user_id", "score"]
        )
        df = df.rename_axis(["algorithm", "timestamp", "metric"])
        result = df.groupby(["algorithm", "metric"])["score"].mean().to_frame()
        result["num_user"] = df.groupby(["algorithm", "metric"])["score"].count()
        result = result.rename(columns={"score": "micro_score"})
        return result

    def df_metric(
        self,
        filter_timestamp: None | int = None,
        filter_algo: None | str = None,
        level: MetricLevelEnum = MetricLevelEnum.MACRO,
    ) -> pd.DataFrame:
        """Get DataFrame representation of metrics.

        Returns a DataFrame representation of the metrics. The DataFrame can be
        filtered based on algorithm name and timestamp.

        Args:
            filter_timestamp: Timestamp value to filter on. Defaults to None.
            filter_algo: Algorithm name to filter on. Defaults to None.
            level: Level of the metric to compute. Defaults to MetricLevelEnum.MACRO.

        Returns:
            DataFrame representation of the metrics.
        """
        if level == MetricLevelEnum.MACRO:
            df = self.df_macro_level_metric()
        elif level == MetricLevelEnum.MICRO:
            df = self.df_micro_level_metric()
        elif level == MetricLevelEnum.WINDOW:
            df = self.df_window_level_metric()
        elif level == MetricLevelEnum.USER:
            df = self.df_user_level_metric()
        else:
            raise ValueError("Invalid level specified")

        if filter_algo:
            df = df.filter(like=filter_algo, axis=0)
        if filter_timestamp:
            df = df.filter(like=f"t={filter_timestamp}", axis=0)
        return df

acc = defaultdict(dict) instance-attribute

user_level_metrics property

window_level_metrics property

add(metric, algorithm_name)

Add a metric to the accumulator.

Takes a Metric object and adds it under the algorithm name. If the specified metric already exists for the algorithm, it will be overwritten with the new metric.

Parameters:

Name Type Description Default
metric Metric

Metric to store.

required
algorithm_name str

Name of the algorithm.

required
Source code in src/recnexteval/evaluators/core/accumulator.py
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
def add(self, metric: Metric, algorithm_name: str) -> None:
    """Add a metric to the accumulator.

    Takes a Metric object and adds it under the algorithm name. If
    the specified metric already exists for the algorithm, it will be
    overwritten with the new metric.

    Args:
        metric: Metric to store.
        algorithm_name: Name of the algorithm.
    """
    if metric.identifier in self.acc[algorithm_name]:
        logger.warning(
            f"Metric {metric.identifier} already exists for algorithm {algorithm_name}. Overwriting..."
        )

    logger.debug(f"Metric {metric.identifier} created for algorithm {algorithm_name}")

    self.acc[algorithm_name][metric.identifier] = metric

df_user_level_metric()

Get user-level metrics across all timestamps.

Returns:

Type Description
DataFrame

DataFrame with user-level metric computations.

Source code in src/recnexteval/evaluators/core/accumulator.py
78
79
80
81
82
83
84
85
86
87
88
89
def df_user_level_metric(self) -> pd.DataFrame:
    """Get user-level metrics across all timestamps.

    Returns:
        DataFrame with user-level metric computations.
    """
    df = pd.DataFrame.from_dict(self.user_level_metrics, orient="index").explode(
        ["user_id", "score"]
    )
    df = df.rename_axis(["algorithm", "timestamp", "metric"])
    df.rename(columns={"score": "user_score"}, inplace=True)
    return df

df_window_level_metric()

Source code in src/recnexteval/evaluators/core/accumulator.py
91
92
93
94
95
96
97
def df_window_level_metric(self) -> pd.DataFrame:
    df = pd.DataFrame.from_dict(self.window_level_metrics, orient="index").explode(
        ["score", "num_user"]
    )
    df = df.rename_axis(["algorithm", "timestamp", "metric"])
    df.rename(columns={"score": "window_score"}, inplace=True)
    return df

df_macro_level_metric()

Get macro-level metrics across all timestamps.

Returns:

Type Description
DataFrame

DataFrame with macro-level metric computations.

Source code in src/recnexteval/evaluators/core/accumulator.py
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
def df_macro_level_metric(self) -> pd.DataFrame:
    """Get macro-level metrics across all timestamps.

    Returns:
        DataFrame with macro-level metric computations.
    """
    df = pd.DataFrame.from_dict(self.window_level_metrics, orient="index").explode(
        ["score", "num_user"]
    )
    df = df.rename_axis(["algorithm", "timestamp", "metric"])
    result = df.groupby(["algorithm", "metric"]).mean()["score"].to_frame()
    result["num_window"] = df.groupby(["algorithm", "metric"]).count()["score"]
    result = result.rename(columns={"score": "macro_score"})
    return result

df_micro_level_metric()

Get micro-level metrics across all timestamps.

Returns:

Type Description
DataFrame

DataFrame with micro-level metric computations.

Source code in src/recnexteval/evaluators/core/accumulator.py
114
115
116
117
118
119
120
121
122
123
124
125
126
127
def df_micro_level_metric(self) -> pd.DataFrame:
    """Get micro-level metrics across all timestamps.

    Returns:
        DataFrame with micro-level metric computations.
    """
    df = pd.DataFrame.from_dict(self.user_level_metrics, orient="index").explode(
        ["user_id", "score"]
    )
    df = df.rename_axis(["algorithm", "timestamp", "metric"])
    result = df.groupby(["algorithm", "metric"])["score"].mean().to_frame()
    result["num_user"] = df.groupby(["algorithm", "metric"])["score"].count()
    result = result.rename(columns={"score": "micro_score"})
    return result

df_metric(filter_timestamp=None, filter_algo=None, level=MetricLevelEnum.MACRO)

Get DataFrame representation of metrics.

Returns a DataFrame representation of the metrics. The DataFrame can be filtered based on algorithm name and timestamp.

Parameters:

Name Type Description Default
filter_timestamp None | int

Timestamp value to filter on. Defaults to None.

None
filter_algo None | str

Algorithm name to filter on. Defaults to None.

None
level MetricLevelEnum

Level of the metric to compute. Defaults to MetricLevelEnum.MACRO.

MACRO

Returns:

Type Description
DataFrame

DataFrame representation of the metrics.

Source code in src/recnexteval/evaluators/core/accumulator.py
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
def df_metric(
    self,
    filter_timestamp: None | int = None,
    filter_algo: None | str = None,
    level: MetricLevelEnum = MetricLevelEnum.MACRO,
) -> pd.DataFrame:
    """Get DataFrame representation of metrics.

    Returns a DataFrame representation of the metrics. The DataFrame can be
    filtered based on algorithm name and timestamp.

    Args:
        filter_timestamp: Timestamp value to filter on. Defaults to None.
        filter_algo: Algorithm name to filter on. Defaults to None.
        level: Level of the metric to compute. Defaults to MetricLevelEnum.MACRO.

    Returns:
        DataFrame representation of the metrics.
    """
    if level == MetricLevelEnum.MACRO:
        df = self.df_macro_level_metric()
    elif level == MetricLevelEnum.MICRO:
        df = self.df_micro_level_metric()
    elif level == MetricLevelEnum.WINDOW:
        df = self.df_window_level_metric()
    elif level == MetricLevelEnum.USER:
        df = self.df_user_level_metric()
    else:
        raise ValueError("Invalid level specified")

    if filter_algo:
        df = df.filter(like=filter_algo, axis=0)
    if filter_timestamp:
        df = df.filter(like=f"t={filter_timestamp}", axis=0)
    return df

EvaluatorBase dataclass

Base class for evaluator.

Provides the common methods and attributes for the evaluator classes. Should there be a need to create a new evaluator, it should inherit from this class.

Parameters:

Name Type Description Default
metric_entries list[MetricEntry]

List of metric entries to compute.

required
setting Setting

Setting object.

required
metric_k int

Value of K for the metrics.

required
ignore_unknown_user bool

Ignore unknown users, defaults to False.

False
ignore_unknown_item bool

Ignore unknown items, defaults to False.

False
seed int

Random seed for reproducibility.

42
Source code in src/recnexteval/evaluators/core/base.py
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
@dataclass
class EvaluatorBase:
    """Base class for evaluator.

    Provides the common methods and attributes for the evaluator classes. Should
    there be a need to create a new evaluator, it should inherit from this class.

    Args:
        metric_entries: List of metric entries to compute.
        setting: Setting object.
        metric_k: Value of K for the metrics.
        ignore_unknown_user: Ignore unknown users, defaults to False.
        ignore_unknown_item: Ignore unknown items, defaults to False.
        seed: Random seed for reproducibility.
    """

    metric_entries: list[MetricEntry]
    setting: Setting
    metric_k: int
    ignore_unknown_user: bool = False
    ignore_unknown_item: bool = False
    seed: int = 42
    user_item_base: UserItemKnowledgeBase = field(default_factory=UserItemKnowledgeBase)
    _run_step: int = 0
    _acc: MetricAccumulator = field(init=False)
    _current_timestamp: int = field(init=False)

    def _get_training_data(self) -> PredictionMatrix:
        if self._run_step == 0:
            logger.debug("First step, getting training data")
            training_data = self.setting.training_data
        else:
            logger.debug("Not first step, getting previous ground truth data as training data")
            training_data = self.setting.get_split_at(self._run_step).incremental
            if training_data is None:
                raise ValueError("Incremental data is None in sliding window setting")
            self.user_item_base.reset_unknown_user_item_base()
        self.user_item_base.update_known_user_item_base(training_data)
        training_data = PredictionMatrix.from_interaction_matrix(training_data)
        training_data.mask_user_item_shape(shape=self.user_item_base.known_shape)
        return training_data

    def _get_evaluation_data(self) -> tuple[PredictionMatrix, PredictionMatrix, int]:
        """Get the evaluation data for the current step.

        Internal method to get the evaluation data for the current step. The
        evaluation data consists of the unlabeled data, ground truth data, and
        the current timestamp which will be returned as a tuple. The shapes
        are masked based through `user_item_base`. The unknown users in
        the ground truth data are also updated in `user_item_base`.

        Note:
            `_current_timestamp` is updated with the current timestamp.

        Returns:
            Tuple of unlabeled data, ground truth data, and current timestamp.

        Raises:
            EOWSettingError: If there is no more data to be processed.
        """
        try:
            split = self.setting.get_split_at(self._run_step)
            unlabeled_data = split.unlabeled
            ground_truth_data = split.ground_truth
            if split.t_window is None:
                raise ValueError("Timestamp of the current split cannot be None")
            self._current_timestamp = split.t_window

            unlabeled_data = PredictionMatrix.from_interaction_matrix(unlabeled_data)
            ground_truth_data = PredictionMatrix.from_interaction_matrix(ground_truth_data)
            self._run_step += 1
        except EOWSettingError:
            raise EOWSettingError("There is no more data to be processed, EOW reached")

        self.user_item_base.update_unknown_user_item_base(ground_truth_data)
        mask_shape = self.user_item_base.global_shape

        if self.ignore_unknown_item:
            # get the unknown items from our knowledge base
            # drop all rows with unknown items in ground truth
            # drop corresponding rows in unlabeled data
            ground_truth_data = ground_truth_data.items_in(self.user_item_base.known_item)
            mask_shape = (mask_shape[0], self.user_item_base.known_shape[1])
        if self.ignore_unknown_user:
            # get the unknown users from our knowledge base
            # drop all columns with unknown users in ground truth
            # drop corresponding columns in unlabeled data
            ground_truth_data = ground_truth_data.users_in(self.user_item_base.known_user)
            mask_shape = (self.user_item_base.known_shape[0], mask_shape[1])
        unlabeled_data._df = unlabeled_data._df.loc[ground_truth_data._df.index]

        unlabeled_data.mask_user_item_shape(shape=mask_shape)
        ground_truth_data.mask_user_item_shape(shape=mask_shape)
        return unlabeled_data, ground_truth_data, self._current_timestamp

    def _add_metric_results_for_prediction(
            self,
            ground_truth_data: PredictionMatrix,
            y_pred: csr_matrix,
            algorithm_name: str,
        ) -> None:
        for metric_entry in self.metric_entries:
            metric_cls = METRIC_REGISTRY.get(metric_entry.name)
            params = {
                'timestamp_limit': self._current_timestamp,
                'user_id_sequence_array': ground_truth_data.user_id_sequence_array,
                'user_item_shape': ground_truth_data.user_item_shape,
            }
            if metric_entry.K is not None:
                params['K'] = metric_entry.K

            metric = metric_cls(**params)
            metric.calculate(y_true=ground_truth_data.item_interaction_sequence_matrix, y_pred=y_pred)
            self._acc.add(metric=metric, algorithm_name=algorithm_name)

    def _prediction_unknown_item_handler(self, y_true: csr_matrix, y_pred: csr_matrix) -> csr_matrix:
        """Handle shape difference due to unknown items in ground truth matrix.

        Extends the number of columns in the prediction matrix to match the ground truth. This is equivalent
        to submitting zero prediction for unknown items.
        """
        if y_pred.shape[1] == y_true.shape[1]:
            return y_pred
        logger.warning(
            "Prediction matrix shape %s is different from ground truth matrix shape %s.",
            y_pred.shape,
            y_true.shape,
        )

        y_pred = csr_matrix(
            (y_pred.data, y_pred.indices, y_pred.indptr),
            shape=(y_pred.shape[0], y_true.shape[1]),
        )
        return y_pred

    def metric_results(
        self,
        level: MetricLevelEnum | Literal["macro", "micro", "window", "user"] = MetricLevelEnum.MACRO,
        only_current_timestamp: None | bool = False,
        filter_timestamp: None | int = None,
        filter_algo: None | str = None,
    ) -> pd.DataFrame:
        """Results of the metrics computed.

        Computes the metrics of all algorithms based on the level specified and
        return the results in a pandas DataFrame. The results can be filtered
        based on the algorithm name and the current timestamp.

        Specifics
        ---------
        - User level: User level metrics computed across all timestamps.
        - Window level: Window level metrics computed across all timestamps. This can
            be viewed as a macro level metric in the context of a single window, where
            the scores of each user is averaged within the window.
        - Macro level: Macro level metrics computed for entire timeline. This
            score is computed by averaging the scores of all windows, treating each
            window equally.
        - Micro level: Micro level metrics computed for entire timeline. This
            score is computed by averaging the scores of all users, treating each
            user and the timestamp the user is in as unique contribution to the
            overall score.

        Args:
            level: Level of the metric to compute, defaults to "macro".
            only_current_timestamp: Filter only the current timestamp, defaults to False.
            filter_timestamp: Timestamp value to filter on, defaults to None.
                If both `only_current_timestamp` and `filter_timestamp` are provided,
                `filter_timestamp` will be used.
            filter_algo: Algorithm name to filter on, defaults to None.

        Returns:
            Dataframe representation of the metric.
        """
        if isinstance(level, str) and not MetricLevelEnum.has_value(level):
            raise ValueError("Invalid level specified")
        level = MetricLevelEnum(level)

        if only_current_timestamp and filter_timestamp:
            raise ValueError("Cannot specify both only_current_timestamp and filter_timestamp.")

        timestamp = None
        if only_current_timestamp:
            timestamp = self._current_timestamp

        if filter_timestamp:
            timestamp = filter_timestamp

        return self._acc.df_metric(filter_algo=filter_algo, filter_timestamp=timestamp, level=level)

    def plot_macro_level_metric(self) -> None:
        df = self.metric_results("macro")
        df = df.reset_index()
        ax = sns.barplot(
            data=df,
            x="metric",
            y="macro_score",
            hue="algorithm",
            edgecolor="black"
        )

        ax.set_xlabel("Metric")
        ax.set_ylabel("Macro score")
        ax.set_title("Macro-level scores by metric and algorithm")

        for container in ax.containers:
            ax.bar_label(container, fmt='%.4f', padding=3, fontsize=8)
        plt.legend(
            title="Algorithm",
            loc="upper center",
            bbox_to_anchor=(0.5, -0.1),
        )
        ax.grid(axis="y", alpha=0.3, linestyle="--")
        plt.show()

    def plot_micro_level_metric(self) -> None:
        df = self.metric_results("micro")
        df = df.reset_index()
        ax = sns.barplot(
            data=df,
            x="metric",
            y="micro_score",
            hue="algorithm",
            edgecolor="black"
        )

        ax.set_xlabel("Metric")
        ax.set_ylabel("Micro score")
        ax.set_title("Micro-level scores by metric and algorithm")

        for container in ax.containers:
            ax.bar_label(container, fmt='%.4f', padding=3, fontsize=8)
        plt.legend(
            title="Algorithm",
            loc="upper center",
            bbox_to_anchor=(0.5, -0.1),
        )
        ax.grid(axis="y", alpha=0.3, linestyle="--")
        plt.show()

    def plot_window_level_metric(self) -> None:
        df = self.metric_results("window")
        df = df.reset_index()
        metrics = df["metric"].unique()
        n_metrics = len(metrics)

        fig, axes = plt.subplots(n_metrics, 1, figsize=(10, 7), sharey=False)
        if n_metrics == 1:
            axes = [axes]

        fig.suptitle("Window-level scores over time", fontsize=14, fontweight="bold")

        for ax, metric in zip(axes, metrics):
            # Filter data for this metric
            metric_df = df[df["metric"] == metric]

            # Plot line for each algorithm
            sns.lineplot(
                data=metric_df,
                x="timestamp",
                y="window_score",
                hue="algorithm",
                marker="o",
                markersize=6,
                linewidth=2,
                ax=ax,
            )
            ax.set_xlabel("Timestamp (epoch)")
            ax.set_ylabel(f"{metric} score")
            ax.grid(axis="both", alpha=0.3, linestyle="--")

            # Remove individual legends
            if ax.get_legend() is not None:
                ax.get_legend().remove()

        # Create single shared legend at bottom
        handles, labels = axes[0].get_legend_handles_labels()

        fig.legend(
            handles,
            labels,
            title="Algorithm",
            loc="lower center",
            bbox_to_anchor=(0.5, -0.15),
            ncol=1,  # vertical stacking
            frameon=True,
            fontsize=9,
        )
        plt.show()

    def restore(self) -> None:
        """Restore the generators before pickling.

        This method is used to restore the generators after loading the object
        from a pickle file.
        """
        self.setting.restore(self._run_step)
        logger.debug("Generators restored")

    def current_step(self) -> int:
        """Return the current step of the evaluator.

        Returns:
            Current step of the evaluator.
        """
        return self._run_step

metric_entries instance-attribute

setting instance-attribute

metric_k instance-attribute

ignore_unknown_user = False class-attribute instance-attribute

ignore_unknown_item = False class-attribute instance-attribute

seed = 42 class-attribute instance-attribute

user_item_base = field(default_factory=UserItemKnowledgeBase) class-attribute instance-attribute

metric_results(level=MetricLevelEnum.MACRO, only_current_timestamp=False, filter_timestamp=None, filter_algo=None)

Results of the metrics computed.

Computes the metrics of all algorithms based on the level specified and return the results in a pandas DataFrame. The results can be filtered based on the algorithm name and the current timestamp.

Specifics
  • User level: User level metrics computed across all timestamps.
  • Window level: Window level metrics computed across all timestamps. This can be viewed as a macro level metric in the context of a single window, where the scores of each user is averaged within the window.
  • Macro level: Macro level metrics computed for entire timeline. This score is computed by averaging the scores of all windows, treating each window equally.
  • Micro level: Micro level metrics computed for entire timeline. This score is computed by averaging the scores of all users, treating each user and the timestamp the user is in as unique contribution to the overall score.

Parameters:

Name Type Description Default
level MetricLevelEnum | Literal['macro', 'micro', 'window', 'user']

Level of the metric to compute, defaults to "macro".

MACRO
only_current_timestamp None | bool

Filter only the current timestamp, defaults to False.

False
filter_timestamp None | int

Timestamp value to filter on, defaults to None. If both only_current_timestamp and filter_timestamp are provided, filter_timestamp will be used.

None
filter_algo None | str

Algorithm name to filter on, defaults to None.

None

Returns:

Type Description
DataFrame

Dataframe representation of the metric.

Source code in src/recnexteval/evaluators/core/base.py
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
def metric_results(
    self,
    level: MetricLevelEnum | Literal["macro", "micro", "window", "user"] = MetricLevelEnum.MACRO,
    only_current_timestamp: None | bool = False,
    filter_timestamp: None | int = None,
    filter_algo: None | str = None,
) -> pd.DataFrame:
    """Results of the metrics computed.

    Computes the metrics of all algorithms based on the level specified and
    return the results in a pandas DataFrame. The results can be filtered
    based on the algorithm name and the current timestamp.

    Specifics
    ---------
    - User level: User level metrics computed across all timestamps.
    - Window level: Window level metrics computed across all timestamps. This can
        be viewed as a macro level metric in the context of a single window, where
        the scores of each user is averaged within the window.
    - Macro level: Macro level metrics computed for entire timeline. This
        score is computed by averaging the scores of all windows, treating each
        window equally.
    - Micro level: Micro level metrics computed for entire timeline. This
        score is computed by averaging the scores of all users, treating each
        user and the timestamp the user is in as unique contribution to the
        overall score.

    Args:
        level: Level of the metric to compute, defaults to "macro".
        only_current_timestamp: Filter only the current timestamp, defaults to False.
        filter_timestamp: Timestamp value to filter on, defaults to None.
            If both `only_current_timestamp` and `filter_timestamp` are provided,
            `filter_timestamp` will be used.
        filter_algo: Algorithm name to filter on, defaults to None.

    Returns:
        Dataframe representation of the metric.
    """
    if isinstance(level, str) and not MetricLevelEnum.has_value(level):
        raise ValueError("Invalid level specified")
    level = MetricLevelEnum(level)

    if only_current_timestamp and filter_timestamp:
        raise ValueError("Cannot specify both only_current_timestamp and filter_timestamp.")

    timestamp = None
    if only_current_timestamp:
        timestamp = self._current_timestamp

    if filter_timestamp:
        timestamp = filter_timestamp

    return self._acc.df_metric(filter_algo=filter_algo, filter_timestamp=timestamp, level=level)

plot_macro_level_metric()

Source code in src/recnexteval/evaluators/core/base.py
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
def plot_macro_level_metric(self) -> None:
    df = self.metric_results("macro")
    df = df.reset_index()
    ax = sns.barplot(
        data=df,
        x="metric",
        y="macro_score",
        hue="algorithm",
        edgecolor="black"
    )

    ax.set_xlabel("Metric")
    ax.set_ylabel("Macro score")
    ax.set_title("Macro-level scores by metric and algorithm")

    for container in ax.containers:
        ax.bar_label(container, fmt='%.4f', padding=3, fontsize=8)
    plt.legend(
        title="Algorithm",
        loc="upper center",
        bbox_to_anchor=(0.5, -0.1),
    )
    ax.grid(axis="y", alpha=0.3, linestyle="--")
    plt.show()

plot_micro_level_metric()

Source code in src/recnexteval/evaluators/core/base.py
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
def plot_micro_level_metric(self) -> None:
    df = self.metric_results("micro")
    df = df.reset_index()
    ax = sns.barplot(
        data=df,
        x="metric",
        y="micro_score",
        hue="algorithm",
        edgecolor="black"
    )

    ax.set_xlabel("Metric")
    ax.set_ylabel("Micro score")
    ax.set_title("Micro-level scores by metric and algorithm")

    for container in ax.containers:
        ax.bar_label(container, fmt='%.4f', padding=3, fontsize=8)
    plt.legend(
        title="Algorithm",
        loc="upper center",
        bbox_to_anchor=(0.5, -0.1),
    )
    ax.grid(axis="y", alpha=0.3, linestyle="--")
    plt.show()

plot_window_level_metric()

Source code in src/recnexteval/evaluators/core/base.py
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
def plot_window_level_metric(self) -> None:
    df = self.metric_results("window")
    df = df.reset_index()
    metrics = df["metric"].unique()
    n_metrics = len(metrics)

    fig, axes = plt.subplots(n_metrics, 1, figsize=(10, 7), sharey=False)
    if n_metrics == 1:
        axes = [axes]

    fig.suptitle("Window-level scores over time", fontsize=14, fontweight="bold")

    for ax, metric in zip(axes, metrics):
        # Filter data for this metric
        metric_df = df[df["metric"] == metric]

        # Plot line for each algorithm
        sns.lineplot(
            data=metric_df,
            x="timestamp",
            y="window_score",
            hue="algorithm",
            marker="o",
            markersize=6,
            linewidth=2,
            ax=ax,
        )
        ax.set_xlabel("Timestamp (epoch)")
        ax.set_ylabel(f"{metric} score")
        ax.grid(axis="both", alpha=0.3, linestyle="--")

        # Remove individual legends
        if ax.get_legend() is not None:
            ax.get_legend().remove()

    # Create single shared legend at bottom
    handles, labels = axes[0].get_legend_handles_labels()

    fig.legend(
        handles,
        labels,
        title="Algorithm",
        loc="lower center",
        bbox_to_anchor=(0.5, -0.15),
        ncol=1,  # vertical stacking
        frameon=True,
        fontsize=9,
    )
    plt.show()

restore()

Restore the generators before pickling.

This method is used to restore the generators after loading the object from a pickle file.

Source code in src/recnexteval/evaluators/core/base.py
310
311
312
313
314
315
316
317
def restore(self) -> None:
    """Restore the generators before pickling.

    This method is used to restore the generators after loading the object
    from a pickle file.
    """
    self.setting.restore(self._run_step)
    logger.debug("Generators restored")

current_step()

Return the current step of the evaluator.

Returns:

Type Description
int

Current step of the evaluator.

Source code in src/recnexteval/evaluators/core/base.py
319
320
321
322
323
324
325
def current_step(self) -> int:
    """Return the current step of the evaluator.

    Returns:
        Current step of the evaluator.
    """
    return self._run_step

AlgorithmStateEnum

Bases: StrEnum

Enum for the state of the algorithm.

Used to keep track of the state of the algorithm during the streaming process in the EvaluatorStreamer.

Source code in src/recnexteval/evaluators/core/constant.py
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
class AlgorithmStateEnum(StrEnum):
    """Enum for the state of the algorithm.

    Used to keep track of the state of the algorithm during the streaming
    process in the `EvaluatorStreamer`.
    """

    NEW = "NEW"
    READY = "READY"
    RUNNING = "RUNNING"
    PREDICTED = "PREDICTED"
    COMPLETED = "COMPLETED"

NEW = 'NEW' class-attribute instance-attribute

READY = 'READY' class-attribute instance-attribute

RUNNING = 'RUNNING' class-attribute instance-attribute

PREDICTED = 'PREDICTED' class-attribute instance-attribute

COMPLETED = 'COMPLETED' class-attribute instance-attribute

MetricLevelEnum

Bases: StrEnum

Source code in src/recnexteval/evaluators/core/constant.py
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
class MetricLevelEnum(StrEnum):
    MICRO = "micro"
    MACRO = "macro"
    WINDOW = "window"
    USER = "user"

    @classmethod
    def has_value(cls, value: str) -> bool:
        """Check valid value for MetricLevelEnum.

        Args:
            value: String value input.

        Returns:
            Whether the value is valid.
        """
        return value in MetricLevelEnum

MICRO = 'micro' class-attribute instance-attribute

MACRO = 'macro' class-attribute instance-attribute

WINDOW = 'window' class-attribute instance-attribute

USER = 'user' class-attribute instance-attribute

has_value(value) classmethod

Check valid value for MetricLevelEnum.

Parameters:

Name Type Description Default
value str

String value input.

required

Returns:

Type Description
bool

Whether the value is valid.

Source code in src/recnexteval/evaluators/core/constant.py
24
25
26
27
28
29
30
31
32
33
34
@classmethod
def has_value(cls, value: str) -> bool:
    """Check valid value for MetricLevelEnum.

    Args:
        value: String value input.

    Returns:
        Whether the value is valid.
    """
    return value in MetricLevelEnum

AlgorithmStateEntry dataclass

Entry for the algorithm status registry.

This dataclass stores the status of an algorithm for use by AlgorithmStateManager. It contains the algorithm name, unique identifier, current state, associated data segment, and an optional pointer to the algorithm object.

Attributes:

Name Type Description
name str

Name of the algorithm.

algorithm_uuid UUID

Unique identifier for the algorithm.

algorithm_ptr Algorithm

Pointer to the algorithm object.

state AlgorithmStateEnum

State of the algorithm.

data_segment int

Data segment the algorithm is associated with.

params dict[str, Any]

Parameters for the algorithm.

Source code in src/recnexteval/evaluators/core/state_management.py
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
@dataclass
class AlgorithmStateEntry:
    """Entry for the algorithm status registry.

    This dataclass stores the status of an algorithm for use by
    `AlgorithmStateManager`. It contains the algorithm name, unique
    identifier, current state, associated data segment, and an optional
    pointer to the algorithm object.

    Attributes:
        name: Name of the algorithm.
        algorithm_uuid: Unique identifier for the algorithm.
        algorithm_ptr: Pointer to the algorithm object.
        state: State of the algorithm.
        data_segment: Data segment the algorithm is associated with.
        params: Parameters for the algorithm.
    """

    name: str
    algorithm_uuid: UUID
    algorithm_ptr: Algorithm
    state: AlgorithmStateEnum = AlgorithmStateEnum.NEW
    data_segment: int = 0
    params: dict[str, Any] = field(default_factory=dict)

name instance-attribute

algorithm_uuid instance-attribute

algorithm_ptr instance-attribute

state = AlgorithmStateEnum.NEW class-attribute instance-attribute

data_segment = 0 class-attribute instance-attribute

params = field(default_factory=dict) class-attribute instance-attribute

AlgorithmStateManager

Source code in src/recnexteval/evaluators/core/state_management.py
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
class AlgorithmStateManager:
    def __init__(self) -> None:
        self._algorithms: dict[UUID, AlgorithmStateEntry] = {}

    def __iter__(self) -> Iterator[UUID]:
        """Return an iterator over registered algorithm UUIDs.

        Allows iteration over the UUIDs of registered entries.

        Returns:
            An iterator over the UUIDs of registered entries.
        """
        return iter(self._algorithms)

    def __len__(self) -> int:
        """Return the number of registered algorithms.

        Returns:
            The number of registered algorithms.
        """
        return len(self._algorithms)

    def values(self) -> Iterator[AlgorithmStateEntry]:
        """Return an iterator over registered AlgorithmStateEntry objects.

        Allows iteration over the registered entries.

        Returns:
            An iterator over the registered entries.
        """
        return iter(self._algorithms.values())

    def __getitem__(self, key: UUID) -> AlgorithmStateEntry:
        if key not in self._algorithms:
            raise ValueError(f"Algorithm with ID:{key} not registered")
        return self._algorithms[key]

    def __setitem__(self, key: UUID, entry: AlgorithmStateEntry) -> None:
        """Register a new algorithm status entry under `key`.

        Allows the use of square bracket notation to register new entries.

        Args:
            key: The UUID to register the entry under.
            entry: The status entry to register.

        Raises:
            KeyError: If `key` is already registered.
        """
        if key in self:
            raise KeyError(f"Algorithm with ID:{key} already registered")
        self._algorithms[key] = entry

    def __contains__(self, key: UUID) -> bool:
        """Return whether the given key is known to the registry.

        Args:
            key: The key to check.

        Returns:
            True if the key is registered, False otherwise.
        """
        try:
            self[key]
            return True
        except AttributeError:
            return False

    def get(self, algo_id: UUID) -> AlgorithmStateEntry:
        """Get the :class:`AlgorithmStateEntry` for `algo_id`."""
        return self[algo_id]

    def get_state(self, algo_id: UUID) -> AlgorithmStateEnum:
        """Get the current state of the algorithm with `algo_id`."""
        return self[algo_id].state

    def register(
        self,
        algorithm_ptr: type[Algorithm] | Algorithm,
        name: None | str = None,
        params: dict[str, Any] = {},
        algo_uuid: None | UUID = None,
    ) -> UUID:
        """Register new algorithm"""
        if isinstance(algorithm_ptr, type):
            algorithm_ptr = algorithm_ptr(**params)

        if hasattr(algorithm_ptr, "identifier"):
            name = name or algorithm_ptr.identifier  # type: ignore[attr-defined]

        if not name:
            logger.warning("Algorithm name was not provided and could not be inferred from Algorithm pointer")
            name = "UnknownAlgorithm"

        if algo_uuid is None:
            algo_uuid = generate_algorithm_uuid(name=name)

        entry = AlgorithmStateEntry(algorithm_uuid=algo_uuid, name=name, algorithm_ptr=algorithm_ptr, params=params)
        self._algorithms[algo_uuid] = entry
        logger.info(f"Registered algorithm '{name}' with ID {algo_uuid}")
        return algo_uuid

    def can_request_training_data(self, algo_id: UUID) -> tuple[bool, str]:
        """Check if algorithm can request training data"""
        if algo_id not in self._algorithms:
            return False, f"Algorithm {algo_id} not registered"

        state = self._algorithms[algo_id].state

        if state == AlgorithmStateEnum.COMPLETED:
            return False, "Algorithm has completed evaluation"
        if state == AlgorithmStateEnum.NEW:
            return False, "The algorithm must be set to READY state first"
        if state == AlgorithmStateEnum.PREDICTED:
            return False, "Algorithm has already requested data for this window"
        if state == AlgorithmStateEnum.READY:
            return True, ""
        if state == AlgorithmStateEnum.RUNNING:
            return True, ""

        return False, f"Unknown state {state}"

    def can_request_unlabeled_data(self, algo_id: UUID) -> tuple[bool, str]:
        """Check if algorithm can request unlabeled data"""
        if algo_id not in self._algorithms:
            return False, f"Algorithm {algo_id} not registered"

        state = self._algorithms[algo_id].state

        if state == AlgorithmStateEnum.RUNNING:
            return True, ""
        if state == AlgorithmStateEnum.COMPLETED:
            return False, "Algorithm has completed evaluation"
        if state == AlgorithmStateEnum.NEW:
            return False, "The algorithm must be set to RUNNING state to request unlabeled data"
        if state == AlgorithmStateEnum.PREDICTED:
            return False, "Algorithm has already requested data for this window"
        if state == AlgorithmStateEnum.READY:
            return (
                False,
                "The algorithm must be set to RUNNING state to request unlabeled data. Request training data first",
            )

        return False, f"Unknown state {state}"

    def can_submit_prediction(self, algo_id: UUID) -> tuple[bool, str]:
        """Check if algorithm can submit prediction"""
        if algo_id not in self._algorithms:
            return False, f"Algorithm {algo_id} not registered"

        state = self._algorithms[algo_id].state

        if state == AlgorithmStateEnum.RUNNING:
            return True, ""
        if state == AlgorithmStateEnum.READY:
            return False, "There is new data to be requested"
        if state == AlgorithmStateEnum.NEW:
            return False, "Algorithm must request data first"
        if state == AlgorithmStateEnum.PREDICTED:
            return False, "Algorithm already submitted prediction for this window"
        if state == AlgorithmStateEnum.COMPLETED:
            return False, "Algorithm has completed evaluation"

        return False, f"Unknown state {state}"

    def transition(self, algo_id: UUID, new_state: AlgorithmStateEnum, data_segment: None | int = None) -> None:
        """Transition algorithm to new state with validation"""
        if algo_id not in self._algorithms:
            raise ValueError(f"Algorithm {algo_id} not registered")

        entry = self._algorithms[algo_id]
        old_state = entry.state

        # Define valid transitions
        valid_transitions = {
            # old_state: [list of valid new_states]
            AlgorithmStateEnum.NEW: [AlgorithmStateEnum.READY, AlgorithmStateEnum.COMPLETED],
            AlgorithmStateEnum.READY: [AlgorithmStateEnum.RUNNING],
            AlgorithmStateEnum.RUNNING: [AlgorithmStateEnum.RUNNING, AlgorithmStateEnum.PREDICTED],
            AlgorithmStateEnum.PREDICTED: [AlgorithmStateEnum.READY, AlgorithmStateEnum.COMPLETED],
            AlgorithmStateEnum.COMPLETED: [],
        }

        if new_state not in valid_transitions.get(old_state, []):
            raise ValueError(f"Invalid transition: {old_state} -> {new_state}")

        entry.state = new_state
        if data_segment is not None:
            entry.data_segment = data_segment

        logger.debug(f"Algorithm '{entry.name}' transitioned {old_state.value} -> {new_state.value}")

    def is_all_predicted(self) -> bool:
        """Return whether every registered algorithm is in PREDICTED state.

        Returns:
            True if all registered entries have state
            `AlgorithmStateEnum.PREDICTED`, False otherwise.
        """
        if not self._algorithms:
            return False
        return all(entry.state == AlgorithmStateEnum.PREDICTED for entry in self._algorithms.values())

    def get_all_states(self) -> dict[str, AlgorithmStateEnum]:
        """Get state of all algorithms"""
        return {entry.name: entry.state for entry in self._algorithms.values()}

    def is_all_same_data_segment(self) -> bool:
        """Return whether all registered entries share the same data segment.

        Returns:
            True if there is exactly one distinct data segment across all
            registered entries, False otherwise.
        """
        data_segments: set[None | int] = set()
        for key in self:
            data_segments.add(self[key].data_segment)
        return len(data_segments) == 1

    def all_algo_states(self) -> dict[str, AlgorithmStateEnum]:
        """Return a mapping of identifier strings to algorithm states.

        The identifier used is "{name}_{uuid}" for each registered entry.

        Returns:
            Mapping from identifier string to the entry's
            :class:`AlgorithmStateEnum`.
        """
        states: dict[str, AlgorithmStateEnum] = {}
        for key in self:
            states[f"{self[key].name}_{key}"] = self[key].state
        return states

    def set_all_ready(self, data_segment: int) -> None:
        """Set all registered algorithms to the READY state.

        Args:
            data_segment: Data segment to assign to every algorithm.
        """
        for key in self:
            self.transition(key, AlgorithmStateEnum.READY, data_segment)

    def get_algorithm_identifier(self, algo_id: UUID) -> str:
        """Return a stable identifier string for the algorithm.

        Args:
            algo_id: UUID of the algorithm.

        Returns:
            Identifier in the format "{name}_{uuid}".

        Raises:
            AttributeError: If `algo_id` is not registered.
        """
        if algo_id not in self._algorithms:
            raise AttributeError(f"Algorithm with ID:{algo_id} not registered")
        return f"{self[algo_id].name}_{algo_id}"

values()

Return an iterator over registered AlgorithmStateEntry objects.

Allows iteration over the registered entries.

Returns:

Type Description
Iterator[AlgorithmStateEntry]

An iterator over the registered entries.

Source code in src/recnexteval/evaluators/core/state_management.py
63
64
65
66
67
68
69
70
71
def values(self) -> Iterator[AlgorithmStateEntry]:
    """Return an iterator over registered AlgorithmStateEntry objects.

    Allows iteration over the registered entries.

    Returns:
        An iterator over the registered entries.
    """
    return iter(self._algorithms.values())

get(algo_id)

Get the :class:AlgorithmStateEntry for algo_id.

Source code in src/recnexteval/evaluators/core/state_management.py
109
110
111
def get(self, algo_id: UUID) -> AlgorithmStateEntry:
    """Get the :class:`AlgorithmStateEntry` for `algo_id`."""
    return self[algo_id]

get_state(algo_id)

Get the current state of the algorithm with algo_id.

Source code in src/recnexteval/evaluators/core/state_management.py
113
114
115
def get_state(self, algo_id: UUID) -> AlgorithmStateEnum:
    """Get the current state of the algorithm with `algo_id`."""
    return self[algo_id].state

register(algorithm_ptr, name=None, params={}, algo_uuid=None)

Register new algorithm

Source code in src/recnexteval/evaluators/core/state_management.py
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
def register(
    self,
    algorithm_ptr: type[Algorithm] | Algorithm,
    name: None | str = None,
    params: dict[str, Any] = {},
    algo_uuid: None | UUID = None,
) -> UUID:
    """Register new algorithm"""
    if isinstance(algorithm_ptr, type):
        algorithm_ptr = algorithm_ptr(**params)

    if hasattr(algorithm_ptr, "identifier"):
        name = name or algorithm_ptr.identifier  # type: ignore[attr-defined]

    if not name:
        logger.warning("Algorithm name was not provided and could not be inferred from Algorithm pointer")
        name = "UnknownAlgorithm"

    if algo_uuid is None:
        algo_uuid = generate_algorithm_uuid(name=name)

    entry = AlgorithmStateEntry(algorithm_uuid=algo_uuid, name=name, algorithm_ptr=algorithm_ptr, params=params)
    self._algorithms[algo_uuid] = entry
    logger.info(f"Registered algorithm '{name}' with ID {algo_uuid}")
    return algo_uuid

can_request_training_data(algo_id)

Check if algorithm can request training data

Source code in src/recnexteval/evaluators/core/state_management.py
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
def can_request_training_data(self, algo_id: UUID) -> tuple[bool, str]:
    """Check if algorithm can request training data"""
    if algo_id not in self._algorithms:
        return False, f"Algorithm {algo_id} not registered"

    state = self._algorithms[algo_id].state

    if state == AlgorithmStateEnum.COMPLETED:
        return False, "Algorithm has completed evaluation"
    if state == AlgorithmStateEnum.NEW:
        return False, "The algorithm must be set to READY state first"
    if state == AlgorithmStateEnum.PREDICTED:
        return False, "Algorithm has already requested data for this window"
    if state == AlgorithmStateEnum.READY:
        return True, ""
    if state == AlgorithmStateEnum.RUNNING:
        return True, ""

    return False, f"Unknown state {state}"

can_request_unlabeled_data(algo_id)

Check if algorithm can request unlabeled data

Source code in src/recnexteval/evaluators/core/state_management.py
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
def can_request_unlabeled_data(self, algo_id: UUID) -> tuple[bool, str]:
    """Check if algorithm can request unlabeled data"""
    if algo_id not in self._algorithms:
        return False, f"Algorithm {algo_id} not registered"

    state = self._algorithms[algo_id].state

    if state == AlgorithmStateEnum.RUNNING:
        return True, ""
    if state == AlgorithmStateEnum.COMPLETED:
        return False, "Algorithm has completed evaluation"
    if state == AlgorithmStateEnum.NEW:
        return False, "The algorithm must be set to RUNNING state to request unlabeled data"
    if state == AlgorithmStateEnum.PREDICTED:
        return False, "Algorithm has already requested data for this window"
    if state == AlgorithmStateEnum.READY:
        return (
            False,
            "The algorithm must be set to RUNNING state to request unlabeled data. Request training data first",
        )

    return False, f"Unknown state {state}"

can_submit_prediction(algo_id)

Check if algorithm can submit prediction

Source code in src/recnexteval/evaluators/core/state_management.py
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
def can_submit_prediction(self, algo_id: UUID) -> tuple[bool, str]:
    """Check if algorithm can submit prediction"""
    if algo_id not in self._algorithms:
        return False, f"Algorithm {algo_id} not registered"

    state = self._algorithms[algo_id].state

    if state == AlgorithmStateEnum.RUNNING:
        return True, ""
    if state == AlgorithmStateEnum.READY:
        return False, "There is new data to be requested"
    if state == AlgorithmStateEnum.NEW:
        return False, "Algorithm must request data first"
    if state == AlgorithmStateEnum.PREDICTED:
        return False, "Algorithm already submitted prediction for this window"
    if state == AlgorithmStateEnum.COMPLETED:
        return False, "Algorithm has completed evaluation"

    return False, f"Unknown state {state}"

transition(algo_id, new_state, data_segment=None)

Transition algorithm to new state with validation

Source code in src/recnexteval/evaluators/core/state_management.py
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
def transition(self, algo_id: UUID, new_state: AlgorithmStateEnum, data_segment: None | int = None) -> None:
    """Transition algorithm to new state with validation"""
    if algo_id not in self._algorithms:
        raise ValueError(f"Algorithm {algo_id} not registered")

    entry = self._algorithms[algo_id]
    old_state = entry.state

    # Define valid transitions
    valid_transitions = {
        # old_state: [list of valid new_states]
        AlgorithmStateEnum.NEW: [AlgorithmStateEnum.READY, AlgorithmStateEnum.COMPLETED],
        AlgorithmStateEnum.READY: [AlgorithmStateEnum.RUNNING],
        AlgorithmStateEnum.RUNNING: [AlgorithmStateEnum.RUNNING, AlgorithmStateEnum.PREDICTED],
        AlgorithmStateEnum.PREDICTED: [AlgorithmStateEnum.READY, AlgorithmStateEnum.COMPLETED],
        AlgorithmStateEnum.COMPLETED: [],
    }

    if new_state not in valid_transitions.get(old_state, []):
        raise ValueError(f"Invalid transition: {old_state} -> {new_state}")

    entry.state = new_state
    if data_segment is not None:
        entry.data_segment = data_segment

    logger.debug(f"Algorithm '{entry.name}' transitioned {old_state.value} -> {new_state.value}")

is_all_predicted()

Return whether every registered algorithm is in PREDICTED state.

Returns:

Type Description
bool

True if all registered entries have state

bool

AlgorithmStateEnum.PREDICTED, False otherwise.

Source code in src/recnexteval/evaluators/core/state_management.py
233
234
235
236
237
238
239
240
241
242
def is_all_predicted(self) -> bool:
    """Return whether every registered algorithm is in PREDICTED state.

    Returns:
        True if all registered entries have state
        `AlgorithmStateEnum.PREDICTED`, False otherwise.
    """
    if not self._algorithms:
        return False
    return all(entry.state == AlgorithmStateEnum.PREDICTED for entry in self._algorithms.values())

get_all_states()

Get state of all algorithms

Source code in src/recnexteval/evaluators/core/state_management.py
244
245
246
def get_all_states(self) -> dict[str, AlgorithmStateEnum]:
    """Get state of all algorithms"""
    return {entry.name: entry.state for entry in self._algorithms.values()}

is_all_same_data_segment()

Return whether all registered entries share the same data segment.

Returns:

Type Description
bool

True if there is exactly one distinct data segment across all

bool

registered entries, False otherwise.

Source code in src/recnexteval/evaluators/core/state_management.py
248
249
250
251
252
253
254
255
256
257
258
def is_all_same_data_segment(self) -> bool:
    """Return whether all registered entries share the same data segment.

    Returns:
        True if there is exactly one distinct data segment across all
        registered entries, False otherwise.
    """
    data_segments: set[None | int] = set()
    for key in self:
        data_segments.add(self[key].data_segment)
    return len(data_segments) == 1

all_algo_states()

Return a mapping of identifier strings to algorithm states.

The identifier used is "{name}_{uuid}" for each registered entry.

Returns:

Type Description
dict[str, AlgorithmStateEnum]

Mapping from identifier string to the entry's

dict[str, AlgorithmStateEnum]

class:AlgorithmStateEnum.

Source code in src/recnexteval/evaluators/core/state_management.py
260
261
262
263
264
265
266
267
268
269
270
271
272
def all_algo_states(self) -> dict[str, AlgorithmStateEnum]:
    """Return a mapping of identifier strings to algorithm states.

    The identifier used is "{name}_{uuid}" for each registered entry.

    Returns:
        Mapping from identifier string to the entry's
        :class:`AlgorithmStateEnum`.
    """
    states: dict[str, AlgorithmStateEnum] = {}
    for key in self:
        states[f"{self[key].name}_{key}"] = self[key].state
    return states

set_all_ready(data_segment)

Set all registered algorithms to the READY state.

Parameters:

Name Type Description Default
data_segment int

Data segment to assign to every algorithm.

required
Source code in src/recnexteval/evaluators/core/state_management.py
274
275
276
277
278
279
280
281
def set_all_ready(self, data_segment: int) -> None:
    """Set all registered algorithms to the READY state.

    Args:
        data_segment: Data segment to assign to every algorithm.
    """
    for key in self:
        self.transition(key, AlgorithmStateEnum.READY, data_segment)

get_algorithm_identifier(algo_id)

Return a stable identifier string for the algorithm.

Parameters:

Name Type Description Default
algo_id UUID

UUID of the algorithm.

required

Returns:

Type Description
str

Identifier in the format "{name}_{uuid}".

Raises:

Type Description
AttributeError

If algo_id is not registered.

Source code in src/recnexteval/evaluators/core/state_management.py
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
def get_algorithm_identifier(self, algo_id: UUID) -> str:
    """Return a stable identifier string for the algorithm.

    Args:
        algo_id: UUID of the algorithm.

    Returns:
        Identifier in the format "{name}_{uuid}".

    Raises:
        AttributeError: If `algo_id` is not registered.
    """
    if algo_id not in self._algorithms:
        raise AttributeError(f"Algorithm with ID:{algo_id} not registered")
    return f"{self[algo_id].name}_{algo_id}"

UserItemKnowledgeBase dataclass

Unknown and known user/item base.

This class is used to store the status of the user and item base. The class stores the known and unknown user and item set. The class also provides methods to update the known and unknown user and item set.

Source code in src/recnexteval/evaluators/core/user_item_knowledge_base.py
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
@dataclass
class UserItemKnowledgeBase:
    """Unknown and known user/item base.

    This class is used to store the status of the user and item base. The class
    stores the known and unknown user and item set. The class also provides
    methods to update the known and unknown user and item set.
    """

    unknown_user: set[int] = field(default_factory=set)
    known_user: set[int] = field(default_factory=set)
    unknown_item: set[int] = field(default_factory=set)
    known_item: set[int] = field(default_factory=set)

    @property
    def known_shape(self) -> tuple[int, int]:
        """Known number of user id and item id.

        id are zero-indexed and the shape returns the max id + 1.

        Note:
            `max` is used over `len` as there may be gaps in the id sequence
            and we are only concerned with the shape of the
            user-item interaction matrix.

        Returns:
            Tuple of (|user|, |item|).
        """
        return (max(self.known_user) + 1, max(self.known_item) + 1)

    @property
    def global_shape(self) -> tuple[int, int]:
        """Global number of user id and item id.

        This is the shape of the user-item interaction matrix considering all
        the users and items that has been possibly exposed. The global shape
        considers the fact that an unknown user/item can be exposed during the
        prediction stage when an unknown user/item id is requested for prediction
        on the algorithm.

        Returns:
            Tuple of (|user|, |item|).
        """
        return (
            max(max(self.known_user), max(self.unknown_user)) + 1,
            max(max(self.known_item), max(self.unknown_item)) + 1,
        )

    def update_known_user_item_base(self, data: InteractionMatrix) -> None:
        """Updates the known user and item set with the data."""
        self.known_item.update(data.item_ids)
        self.known_user.update(data.user_ids)

    def update_unknown_user_item_base(self, data: InteractionMatrix) -> None:
        """Updates the unknown user and item set with the data. """
        self.unknown_user = data.user_ids - self.known_user
        self.unknown_item = data.item_ids - self.known_item

    def reset_unknown_user_item_base(self) -> None:
        """Clears the unknown user and item set."""
        self.unknown_user.clear()
        self.unknown_item.clear()

unknown_user = field(default_factory=set) class-attribute instance-attribute

known_user = field(default_factory=set) class-attribute instance-attribute

unknown_item = field(default_factory=set) class-attribute instance-attribute

known_item = field(default_factory=set) class-attribute instance-attribute

known_shape property

Known number of user id and item id.

id are zero-indexed and the shape returns the max id + 1.

Note

max is used over len as there may be gaps in the id sequence and we are only concerned with the shape of the user-item interaction matrix.

Returns:

Type Description
tuple[int, int]

Tuple of (|user|, |item|).

global_shape property

Global number of user id and item id.

This is the shape of the user-item interaction matrix considering all the users and items that has been possibly exposed. The global shape considers the fact that an unknown user/item can be exposed during the prediction stage when an unknown user/item id is requested for prediction on the algorithm.

Returns:

Type Description
tuple[int, int]

Tuple of (|user|, |item|).

update_known_user_item_base(data)

Updates the known user and item set with the data.

Source code in src/recnexteval/evaluators/core/user_item_knowledge_base.py
58
59
60
61
def update_known_user_item_base(self, data: InteractionMatrix) -> None:
    """Updates the known user and item set with the data."""
    self.known_item.update(data.item_ids)
    self.known_user.update(data.user_ids)

update_unknown_user_item_base(data)

Updates the unknown user and item set with the data.

Source code in src/recnexteval/evaluators/core/user_item_knowledge_base.py
63
64
65
66
def update_unknown_user_item_base(self, data: InteractionMatrix) -> None:
    """Updates the unknown user and item set with the data. """
    self.unknown_user = data.user_ids - self.known_user
    self.unknown_item = data.item_ids - self.known_item

reset_unknown_user_item_base()

Clears the unknown user and item set.

Source code in src/recnexteval/evaluators/core/user_item_knowledge_base.py
68
69
70
71
def reset_unknown_user_item_base(self) -> None:
    """Clears the unknown user and item set."""
    self.unknown_user.clear()
    self.unknown_item.clear()