ranking ndcg (#1461)

## Description

Add ranking ndcg to log_batch_ranking_metrics


- [x] I have reviewed the [Guidelines for Contributing](CONTRIBUTING.md)
and the [Code of Conduct](CODE_OF_CONDUCT.md).

---------

Co-authored-by: Bernease Herman <[email protected]>
Co-authored-by: felipe207 <[email protected]>

Author: 3 people

python/tests/experimental/api/test_logger.py

@@ -1,3 +1,5 @@
+from math import isclose
+
 from whylogs.core.stubs import pd
 from whylogs.experimental.api.logger import log_batch_ranking_metrics
 
@@ -15,37 +17,42 @@ def test_log_batch_ranking_metrics_single_simple():
         }
     )
     result = log_batch_ranking_metrics(
-        k=1, data=single_df, prediction_column="raw_predictions", target_column="raw_targets"
+        data=single_df, prediction_column="raw_predictions", target_column="raw_targets", convert_non_numeric=True
     )
     pandas_summary = result.view().to_pandas()
 
-    k = 1
     column_names = [
-        "mean_average_precision_k_" + str(k),
-        "accuracy_k_" + str(k),
+        "mean_average_precision",
+        "accuracy",
         "mean_reciprocal_rank",
-        "precision_k_" + str(k),
-        "recall_k_" + str(k),
+        "precision",
+        "recall",
         "top_rank",
-        "average_precision_k_" + str(k),
+        "average_precision",
+        "norm_dis_cumul_gain",
     ]
     for col in column_names:
         assert col in pandas_summary.index
-    assert pandas_summary.loc["mean_average_precision_k_" + str(k), "counts/n"] == 1
-    assert pandas_summary.loc["accuracy_k_" + str(k), "counts/n"] == 1
+    assert pandas_summary.loc["mean_average_precision", "counts/n"] == 1
+    assert pandas_summary.loc["accuracy", "counts/n"] == 1
     assert pandas_summary.loc["mean_reciprocal_rank", "counts/n"] == 1
-    assert pandas_summary.loc["precision_k_" + str(k), "counts/n"] == 4
-    assert pandas_summary.loc["recall_k_" + str(k), "counts/n"] == 4
+    assert pandas_summary.loc["precision", "counts/n"] == 4
+    assert pandas_summary.loc["recall", "counts/n"] == 4
     assert pandas_summary.loc["top_rank", "counts/n"] == 4
-    assert pandas_summary.loc["average_precision_k_" + str(k), "counts/n"] == 4
+    assert pandas_summary.loc["average_precision", "counts/n"] == 4
+    assert pandas_summary.loc["norm_dis_cumul_gain", "counts/n"] == 1
+    assert pandas_summary.loc["average_precision", "counts/n"] == 4
+    assert pandas_summary.loc["norm_dis_cumul_gain", "counts/n"] == 1
 
 
 def test_log_batch_ranking_metrics_binary_simple():
     binary_df = pd.DataFrame(
         {"raw_predictions": [[True, False, True], [False, False, False], [True, True, False], [False, True, False]]}
     )
 
-    result = log_batch_ranking_metrics(k=2, data=binary_df, prediction_column="raw_predictions")
+    result = log_batch_ranking_metrics(
+        data=binary_df, prediction_column="raw_predictions", k=2, convert_non_numeric=True
+    )
     pandas_summary = result.view().to_pandas()
 
     k = 2
@@ -57,6 +64,7 @@ def test_log_batch_ranking_metrics_binary_simple():
         "recall_k_" + str(k),
         "top_rank",
         "average_precision_k_" + str(k),
+        "norm_dis_cumul_gain_k_" + str(k),
     ]
     for col in column_names:
         assert col in pandas_summary.index
@@ -67,6 +75,7 @@ def test_log_batch_ranking_metrics_binary_simple():
     assert pandas_summary.loc["recall_k_" + str(k), "counts/n"] == 4
     assert pandas_summary.loc["top_rank", "counts/n"] == 4
     assert pandas_summary.loc["average_precision_k_" + str(k), "counts/n"] == 4
+    assert pandas_summary.loc["norm_dis_cumul_gain_k_" + str(k), "counts/n"] == 1
 
 
 def test_log_batch_ranking_metrics_multiple_simple():
@@ -81,13 +90,17 @@ def test_log_batch_ranking_metrics_multiple_simple():
             ],
         }
     )
+    k = 4
 
     result = log_batch_ranking_metrics(
-        k=3, data=multiple_df, prediction_column="raw_predictions", target_column="raw_targets"
+        data=multiple_df,
+        prediction_column="raw_predictions",
+        target_column="raw_targets",
+        k=k,
+        convert_non_numeric=True,
     )
     pandas_summary = result.view().to_pandas()
 
-    k = 3
     column_names = [
         "mean_average_precision_k_" + str(k),
         "accuracy_k_" + str(k),
@@ -96,6 +109,7 @@ def test_log_batch_ranking_metrics_multiple_simple():
         "recall_k_" + str(k),
         "top_rank",
         "average_precision_k_" + str(k),
+        "norm_dis_cumul_gain_k_" + str(k),
     ]
     for col in column_names:
         assert col in pandas_summary.index
@@ -106,3 +120,67 @@ def test_log_batch_ranking_metrics_multiple_simple():
     assert pandas_summary.loc["recall_k_" + str(k), "counts/n"] == 4
     assert pandas_summary.loc["top_rank", "counts/n"] == 4
     assert pandas_summary.loc["average_precision_k_" + str(k), "counts/n"] == 4
+    assert pandas_summary.loc["norm_dis_cumul_gain_k_" + str(k), "counts/n"] == 1
+
+    assert isclose(pandas_summary.loc[f"norm_dis_cumul_gain_k_{k}", "distribution/median"], 0.76244, abs_tol=0.00001)
+
+
+def test_log_batch_ranking_metrics_default_target():
+    multiple_df = pd.DataFrame({"raw_predictions": [[3, 2, 3, 0, 1, 2, 3, 2]]})
+
+    result = log_batch_ranking_metrics(
+        data=multiple_df, prediction_column="raw_predictions", k=3, convert_non_numeric=True
+    )
+    pandas_summary = result.view().to_pandas()
+
+    k = 3
+    column_names = [
+        "mean_average_precision_k_" + str(k),
+        "accuracy_k_" + str(k),
+        "mean_reciprocal_rank",
+        "precision_k_" + str(k),
+        "recall_k_" + str(k),
+        "top_rank",
+        "average_precision_k_" + str(k),
+        "norm_dis_cumul_gain_k_" + str(k),
+    ]
+    for col in column_names:
+        assert col in pandas_summary.index
+    assert pandas_summary.loc["mean_average_precision_k_" + str(k), "counts/n"] == 1
+    assert pandas_summary.loc["accuracy_k_" + str(k), "counts/n"] == 1
+    assert pandas_summary.loc["mean_reciprocal_rank", "counts/n"] == 1
+    assert pandas_summary.loc["precision_k_" + str(k), "counts/n"] == 1
+    assert pandas_summary.loc["recall_k_" + str(k), "counts/n"] == 1
+    assert pandas_summary.loc["top_rank", "counts/n"] == 1
+    assert pandas_summary.loc["average_precision_k_" + str(k), "counts/n"] == 1
+    assert pandas_summary.loc["norm_dis_cumul_gain_k_" + str(k), "counts/n"] == 1
+
+
+def test_log_batch_ranking_metrics_ranking_ndcg_wikipedia():
+    # From https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Example
+    ranking_df = pd.DataFrame({"targets": [[3, 2, 3, 0, 1, 2, 3, 2]], "predictions": [[7, 6, 5, 4, 3, 2, 1, 0]]})
+
+    result = log_batch_ranking_metrics(data=ranking_df, prediction_column="predictions", target_column="targets", k=6)
+    pandas_summary = result.view().to_pandas()
+
+    assert isclose(pandas_summary.loc["norm_dis_cumul_gain_k_6", "distribution/median"], 0.785, abs_tol=0.01)
+
+
+def test_log_batch_ranking_metrics_ranking_ndcg_sklearn():
+    # From https://scikit-learn.org/stable/modules/generated/sklearn.metrics.ndcg_score.html
+    ranking_df = pd.DataFrame({"predictions": [[0.1, 0.2, 0.3, 4, 70]], "targets": [[10, 0, 0, 1, 5]]})
+
+    result = log_batch_ranking_metrics(data=ranking_df, prediction_column="predictions", target_column="targets")
+    pandas_summary = result.view().to_pandas()
+
+    assert isclose(pandas_summary.loc["norm_dis_cumul_gain", "distribution/median"], 0.69569, abs_tol=0.00001)
+
+
+def test_log_batch_ranking_metrics_ranking_ndcg_withk_sklearn():
+    # From https://scikit-learn.org/stable/modules/generated/sklearn.metrics.ndcg_score.html
+    ranking_df = pd.DataFrame({"predictions": [[0.05, 1.1, 1.0, 0.5, 0.0]], "targets": [[10, 0, 0, 1, 5]]})
+
+    result = log_batch_ranking_metrics(data=ranking_df, prediction_column="predictions", target_column="targets", k=4)
+    pandas_summary = result.view().to_pandas()
+
+    assert isclose(pandas_summary.loc["norm_dis_cumul_gain_k_4", "distribution/median"], 0.35202, abs_tol=0.00001)

python/whylogs/experimental/api/logger/__init__.py

@@ -1,4 +1,5 @@
 import logging
+import math
 from typing import Optional, Union
 
 from whylogs.api.logger import log
@@ -10,11 +11,12 @@
 
 
 def log_batch_ranking_metrics(
-    k: int,
     data: pd.core.frame.DataFrame,
     prediction_column: str,
     target_column: Optional[str] = None,
     score_column: Optional[str] = None,
+    k: Optional[int] = None,
+    convert_non_numeric=False,
     schema: Union[DatasetSchema, None] = None,
     log_full_data: bool = False,
 ) -> ViewResultSet:
@@ -23,8 +25,7 @@ def log_batch_ranking_metrics(
     relevant_cols = [prediction_column]
     if target_column is None:
         target_column = "__targets"
-        formatted_data[target_column] = True
-        formatted_data[target_column].apply(lambda x: [x])
+        formatted_data[target_column] = formatted_data[prediction_column].apply(lambda x: list(range(len(x)))[::-1])
     relevant_cols.append(target_column)
     if score_column is not None:
         relevant_cols.append(score_column)
@@ -35,6 +36,8 @@ def log_batch_ranking_metrics(
             # TODO: more error checking
             formatted_data[col] = formatted_data[col].apply(lambda x: [x])
 
+    _max_k = formatted_data[prediction_column].apply(len).max()
+
     formatted_data["count_at_k"] = formatted_data[relevant_cols].apply(
         lambda row: sum([1 if pred_val in row[target_column] else 0 for pred_val in row[prediction_column][:k]]), axis=1
     )
@@ -52,13 +55,13 @@ def get_top_rank(row):
 
     formatted_data["top_rank"] = formatted_data[relevant_cols].apply(get_top_rank, axis=1)
 
-    output_data = (formatted_data["count_at_k"] / k).to_frame()
-    output_data.columns = ["precision_k_" + str(k)]
-    output_data["recall_k_" + str(k)] = formatted_data["count_at_k"] / formatted_data["count_all"]
+    output_data = (formatted_data["count_at_k"] / (k if k else 1)).to_frame()
+    output_data.columns = ["precision" + ("_k_" + str(k) if k else "")]
+    output_data["recall" + ("_k_" + str(k) if k else "")] = formatted_data["count_at_k"] / formatted_data["count_all"]
     output_data["top_rank"] = formatted_data["top_rank"]
 
     ki_dict: pd.DataFrame = None
-    for ki in range(1, k + 1):
+    for ki in range(1, (k if k else _max_k) + 1):
         ki_result = (
             formatted_data[relevant_cols].apply(
                 lambda row: sum(
@@ -74,18 +77,48 @@ def get_top_rank(row):
         else:
             ki_dict["p@" + str(ki)] = ki_result
 
-    output_data["average_precision_k_" + str(k)] = ki_dict.mean(axis=1)
-    mAP_at_k = output_data["average_precision_k_" + str(k)].mean(axis=0)
+    output_data["average_precision" + ("_k_" + str(k) if k else "")] = ki_dict.mean(axis=1)
+
+    def _convert_non_numeric(row_dict):
+        return (
+            [
+                row_dict[target_column].index(pred_val) if pred_val in row_dict[target_column] else -1
+                for pred_val in row_dict[prediction_column]
+            ],
+            list(range(len(row_dict[prediction_column])))[::-1],
+        )
+
+    if convert_non_numeric:
+        formatted_data[[prediction_column, target_column]] = formatted_data.apply(
+            _convert_non_numeric, result_type="expand", axis=1
+        )
+
+    def _calculate_row_ndcg(row_dict, k):
+        predicted_order = np.array(row_dict[prediction_column]).argsort()[::-1]
+        target_order = np.array(row_dict[target_column]).argsort()[::-1]
+        dcg_vals = [
+            (rel / math.log(i + 2, 2)) for i, rel in enumerate(np.array(row_dict[target_column])[predicted_order][:k])
+        ]
+        idcg_vals = [
+            (rel / math.log(i + 2, 2)) for i, rel in enumerate(np.array(row_dict[target_column])[target_order][:k])
+        ]
+        return sum(dcg_vals) / sum(idcg_vals)
+
+    formatted_data["norm_dis_cumul_gain_k_" + str(k)] = formatted_data.apply(_calculate_row_ndcg, args=(k,), axis=1)
+
+    mAP_at_k = ki_dict.mean()
     hit_ratio = formatted_data["count_at_k"].apply(lambda x: bool(x)).sum() / len(formatted_data)
-    mrr = (1 / output_data["top_rank"]).replace([np.inf], np.nan).mean()
+    mrr = (1 / formatted_data["top_rank"]).replace([np.inf], np.nan).mean()
+    ndcg = formatted_data["norm_dis_cumul_gain_k_" + str(k)].mean()
 
     result = log(pandas=output_data, schema=schema)
     result = result.merge(
         log(
             row={
-                "mean_average_precision_k_" + str(k): mAP_at_k,
-                "accuracy_k_" + str(k): hit_ratio,
+                "mean_average_precision" + ("_k_" + str(k) if k else ""): mAP_at_k,
+                "accuracy" + ("_k_" + str(k) if k else ""): hit_ratio,
                 "mean_reciprocal_rank": mrr,
+                "norm_dis_cumul_gain" + ("_k_" + str(k) if k else ""): ndcg,
             },
             schema=schema,
         )