[Ref, Fix] indentation error in answer key selection, longer explanation in demo, exclusion of broken dataset

app.py

@@ -69,7 +69,7 @@ with gr.Blocks(title="LLM Similarity Analyzer", css=app_util.custom_css) as demo
         outputs=heatmap
     )
     
-    gr.Markdown("\* Self-similarity is only 1.0 for the probabilistic Kappa_p metric if the model predicts a single option with 100% confidence for each question.")
+    gr.Markdown("\* Self-similarity is only 1.0 for CAPA if the model predicts a single option with 100% confidence for each question. If the model is uncertain, the self-similarity will be lower.")
 
     clear_btn = gr.Button("Clear Selection")
     clear_btn.click(

src/dataloading.py

@@ -88,7 +88,7 @@ def get_leaderboard_datasets(model_ids):
         common_datasets = set.intersection(*model_datasets.values())
 
     # Filter datasets that are not MCQ or currently do not work
-    ignore = ["math_", "ifeval"]
+    ignore = ["bbh_temporal_sequences", "math_", "ifeval"]
     discard = []
     for dataset in common_datasets:
         for ignore_data in ignore:
@@ -132,6 +132,8 @@ def filter_labels(dataset_name, doc):
                 labels.append(1)
     elif test_target.isdigit():
         labels = [int(d[target_key]) for d in doc]
+    
+    print(f"Number of labels: {len(labels)}")
 
     return labels
 

src/utils.py

@@ -18,7 +18,9 @@ def opt_in_pars_to_index(s):
         raise ValueError("Invalid format")
     
 def get_test_target(doc):
-     if "target" in doc:
-         return doc["target"], "target"
-     elif "answer" in doc:
-         return doc["answer"], "answer"
+    if "target" in doc:
+        return doc["target"], "target"
+    elif "answer" in doc:
+        return doc["answer"], "answer"
+    else:
+        return "", ""