join Trust & Safety table

src/display/utils.py

@@ -26,35 +26,39 @@ auto_eval_column_dict.append(
     ["model", ColumnContent, ColumnContent("Model Name", "markdown", True, never_hidden=True)]
 )
 auto_eval_column_dict.append(["model_provider", ColumnContent, ColumnContent("LLM Provider", "markdown", True)])
-auto_eval_column_dict.append(["use_case_name", ColumnContent, ColumnContent("Use Case Name", "markdown", True)])
+auto_eval_column_dict.append(["use_case_name", ColumnContent, ColumnContent("Use Case Name", "markdown", False)])
 auto_eval_column_dict.append(["use_case_type", ColumnContent, ColumnContent("Use Case Type", "markdown", False)])
 auto_eval_column_dict.append(["accuracy_method", ColumnContent, ColumnContent("Accuracy Method", "markdown", False)])
 # Accuracy metrics
-auto_eval_column_dict.append(["accuracy_metric_average", ColumnContent, ColumnContent("Accuracy", "markdown", True)])
+auto_eval_column_dict.append(["accuracy_metric_average", ColumnContent, ColumnContent("Accuracy", "markdown", False)])
 auto_eval_column_dict.append(
     [
         "accuracy_metric_instruction_following",
         ColumnContent,
-        ColumnContent("Instruction Following", "markdown", True),
+        ColumnContent("Instruction Following", "markdown", False),
     ]
 )
 auto_eval_column_dict.append(
-    ["accuracy_metric_completeness", ColumnContent, ColumnContent("Completeness", "markdown", True)]
+    ["accuracy_metric_completeness", ColumnContent, ColumnContent("Completeness", "markdown", False)]
 )
 auto_eval_column_dict.append(
-    ["accuracy_metric_conciseness", ColumnContent, ColumnContent("Conciseness", "markdown", True)]
+    ["accuracy_metric_conciseness", ColumnContent, ColumnContent("Conciseness", "markdown", False)]
 )
 auto_eval_column_dict.append(
-    ["accuracy_metric_factuality", ColumnContent, ColumnContent("Factuality", "markdown", True)]
+    ["accuracy_metric_factuality", ColumnContent, ColumnContent("Factuality", "markdown", False)]
 )
-# auto_eval_column_dict.append(
-#     ["use_case_flavor", ColumnContent, ColumnContent("Cost and Speed: Flavor", "markdown", False)]
-# )
+# Speed (Latency) & Cost metrics
 auto_eval_column_dict.append(["latency", ColumnContent, ColumnContent("Response Time (Sec)", "markdown", True)])
 auto_eval_column_dict.append(
     ["mean_output_tokens", ColumnContent, ColumnContent("Mean Output Tokens", "markdown", True)]
 )
 auto_eval_column_dict.append(["cost_band", ColumnContent, ColumnContent("Cost Band", "markdown", True)])
+# Trust & Safety metrics
+auto_eval_column_dict.append(["ts", ColumnContent, ColumnContent("Trust & Safety", "markdown", True)])
+auto_eval_column_dict.append(["safety", ColumnContent, ColumnContent("Safety", "markdown", False)])
+auto_eval_column_dict.append(["privacy", ColumnContent, ColumnContent("Privacy", "markdown", False)])
+auto_eval_column_dict.append(["truthfulness", ColumnContent, ColumnContent("Truthfulness", "markdown", False)])
+auto_eval_column_dict.append(["crm_bias", ColumnContent, ColumnContent("CRM Bias", "markdown", False)])
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
@@ -80,7 +84,7 @@ CostEvalColumn = make_dataclass("CostEvalColumn", cost_eval_column_dict, frozen=
 ts_eval_column_dict = []
 # Init
 ts_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model Name", "markdown", True, never_hidden=True)])
-ts_eval_column_dict.append(["model_provider", ColumnContent, ColumnContent("LLM Provider", "markdown", True)])
+# ts_eval_column_dict.append(["model_provider", ColumnContent, ColumnContent("LLM Provider", "markdown", True)])
 ts_eval_column_dict.append(["ts", ColumnContent, ColumnContent("Trust & Safety", "markdown", True)])
 ts_eval_column_dict.append(["safety", ColumnContent, ColumnContent("Safety", "markdown", False)])
 ts_eval_column_dict.append(["privacy", ColumnContent, ColumnContent("Privacy", "markdown", False)])

src/populate.py

@@ -31,10 +31,9 @@ def get_leaderboard_df_crm(
     )
 
     leaderboard_ts_df = pd.read_csv(os.path.join(crm_results_path, "hf_leaderboard_ts.csv"))
-    leaderboard_ts__crm_bias_df = pd.read_csv(os.path.join(crm_results_path, "hf_leaderboard_crm_bias.csv"))
+    leaderboard_ts_crm_bias_df = pd.read_csv(os.path.join(crm_results_path, "hf_leaderboard_crm_bias.csv"))
     leaderboard_ts_df = leaderboard_ts_df[~leaderboard_ts_df["Model Name"].isin(sf_finetuned_models)]
-    leaderboard_ts_df = leaderboard_ts_df.join(ref_df.set_index("Model Name"), on="Model Name")
-    leaderboard_ts_df = leaderboard_ts_df.join(leaderboard_ts__crm_bias_df.set_index("Model Name"), on="Model Name")
+    leaderboard_ts_df = leaderboard_ts_df.join(leaderboard_ts_crm_bias_df.set_index("Model Name"), on="Model Name")
     privacy_cols = leaderboard_ts_df[
         [
             "Privacy Zero-Shot Match Avoidance",
@@ -47,7 +46,7 @@ def get_leaderboard_df_crm(
     leaderboard_ts_df["Privacy"] = privacy_cols.mean(axis=1).transform(lambda x: "{:,.2%}".format(x))
     leaderboard_ts_df["Bias No CI"] = leaderboard_ts_df["CRM Bias"].transform(lambda x: x.split(" ")[0])
 
-    ts_cols = leaderboard_ts_df[
+    ts_lvl2_cols = leaderboard_ts_df[
         [
             "Safety",
             "Privacy",
@@ -55,7 +54,14 @@ def get_leaderboard_df_crm(
             "Bias No CI",
         ]
     ].apply(lambda x: x.str.rstrip("%").astype("float") / 100.0, axis=1)
-    leaderboard_ts_df["Trust & Safety"] = ts_cols.mean(axis=1).transform(lambda x: "{:,.2%}".format(x))
+    leaderboard_ts_df["Trust & Safety"] = ts_lvl2_cols.mean(axis=1).transform(lambda x: "{:,.2%}".format(x))
+
+    leaderboard_accuracy_df = leaderboard_accuracy_df.join(
+        leaderboard_ts_df[ts_cols].set_index(["Model Name"]),
+        on=["Model Name"],
+    )
+
+    leaderboard_ts_df = leaderboard_ts_df.join(ref_df.set_index("Model Name"), on="Model Name")
 
     leaderboard_accuracy_df = leaderboard_accuracy_df.sort_values(
         by=[AutoEvalColumn.accuracy_metric_average.name], ascending=False