Cross-trait correlation analysis 🔬#
By default we look at agreement between target annotations, representing either a model or human annotations, with personality annotations. In this tutorial we show another way to analyse the traits: how do they correlate with each other?
import pandas as pd
import feedback_forensics as ff
import pathlib
# Load results (e.g. Arena data)
dataset_name = "chatbot_arena.json"
dataset = ff.DatasetHandler()
data_path = pathlib.Path("../../data/output/results_sets/feedback-forensics-results-paper")
dataset.add_data_from_path(data_path / dataset_name)
df = dataset.first_handler.df
annotator_metadata = dataset.get_available_annotators()
metrics = dataset.get_annotator_metrics()
# Get top and bottom 5 annotators according to strength metric
strength_metrics = metrics["chatbot_arena"]["metrics"]["strength"]
annotators = list(strength_metrics.keys())
top_annotators = sorted(annotators, key=lambda x: strength_metrics[x], reverse=True)
top5_annotators = top_annotators[:5]
bottom5_annotators = top_annotators[-5:][::-1]
def get_annotator_key(in_row_name: str) -> str:
for annotator_key, metadata in annotator_metadata.items():
if metadata["annotator_in_row_name"] in in_row_name:
return annotator_key
return None
annotators = {
"top5": {
annotator_name: {"key": get_annotator_key(annotator_name), "name": annotator_name}
for annotator_name in top5_annotators
},
"bottom5": {
annotator_name: {"key": get_annotator_key(annotator_name), "name": annotator_name}
for annotator_name in bottom5_annotators
}
}
full_set_of_annotators = {}
for category, annotator_subset in annotators.items():
for annotator_name in annotator_subset.keys():
annotator_key = annotator_subset[annotator_name]["key"]
annotator_data = df[annotator_key]
annotator_subset[annotator_name]["data"] = annotator_data
full_set_of_annotators[annotator_name] = annotator_data
# create df that underlies correlation analysis
# this df includes the full set of annotations per annotator
corr_df = pd.DataFrame(full_set_of_annotators, dtype="category")
/home/docs/checkouts/readthedocs.org/user_builds/feedback-forensics/envs/latest/lib/python3.11/site-packages/alpaca_eval/utils.py:20: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
import pkg_resources
---------------------------------------------------------------------------
FileNotFoundError Traceback (most recent call last)
Cell In[1], line 9
7 dataset = ff.DatasetHandler()
8 data_path = pathlib.Path("../../data/output/results_sets/feedback-forensics-results-paper")
----> 9 dataset.add_data_from_path(data_path / dataset_name)
10 df = dataset.first_handler.df
12 annotator_metadata = dataset.get_available_annotators()
File ~/checkouts/readthedocs.org/user_builds/feedback-forensics/envs/latest/lib/python3.11/site-packages/feedback_forensics/data/handler.py:352, in DatasetHandler.add_data_from_path(self, path, name)
350 name = str(path).split("/")[-1].split(".")[0]
351 handler = ColumnHandler(cache=self.cache, avail_datasets=self.avail_datasets)
--> 352 handler.load_data_from_path(path)
353 self.add_col_handler(name, handler)
File ~/checkouts/readthedocs.org/user_builds/feedback-forensics/envs/latest/lib/python3.11/site-packages/feedback_forensics/data/handler.py:206, in ColumnHandler.load_data_from_path(self, dataset_path)
203 """Load data from a given path."""
204 dataset_path = Path(dataset_path)
--> 206 base_votes_dict = get_votes_dict(dataset_path, cache=self.cache)
207 votes_dict = add_virtual_annotators(
208 base_votes_dict,
209 cache=self.cache,
(...) 212 target_models=[],
213 )
215 self.load_from_votes_dict(votes_dict)
File ~/checkouts/readthedocs.org/user_builds/feedback-forensics/envs/latest/lib/python3.11/site-packages/feedback_forensics/data/loader.py:56, in get_votes_dict(results_path, cache)
53 cache = {}
55 if not results_path.exists():
---> 56 raise FileNotFoundError(f"Results directory not found in path '{results_path}'")
58 if "votes_dict" in cache and results_path in cache["votes_dict"]:
59 return cache["votes_dict"][results_path]
FileNotFoundError: Results directory not found in path '../../data/output/results_sets/feedback-forensics-results-paper/chatbot_arena.json'
import sklearn.metrics
# create a correlation matrix for all annotators
correlation_matrix = pd.DataFrame(index=corr_df.columns, columns=corr_df.columns)
for annotator_1 in corr_df.columns:
for annotator_2 in corr_df.columns:
# print(f"Comparing {annotator_1} and {annotator_2}")
val = sklearn.metrics.cohen_kappa_score(
corr_df[annotator_1].to_numpy(dtype="str"),
corr_df[annotator_2].to_numpy(dtype="str"),
)
correlation_matrix.loc[annotator_1, annotator_2] = f"{val:.2f}"
correlation_matrix
#correlation_matrix.to_markdown("correlation_matrix.md")
| is more verbose | has more structured formatting | makes more confident statements | is more factually correct | more strictly follows the requested output format | is more concise | has a more avoidant tone | refuses to answer the question | ends with a follow-up question | is more polite | |
|---|---|---|---|---|---|---|---|---|---|---|
| is more verbose | 1.00 | 0.44 | 0.13 | 0.07 | 0.07 | -0.99 | -0.02 | -0.02 | 0.01 | 0.12 |
| has more structured formatting | 0.44 | 1.00 | 0.24 | 0.18 | 0.12 | -0.44 | -0.03 | -0.02 | -0.00 | 0.21 |
| makes more confident statements | 0.13 | 0.24 | 1.00 | 0.55 | 0.12 | -0.13 | 0.03 | 0.02 | 0.01 | 0.17 |
| is more factually correct | 0.07 | 0.18 | 0.55 | 1.00 | 0.10 | -0.07 | 0.06 | 0.05 | -0.01 | 0.11 |
| more strictly follows the requested output format | 0.07 | 0.12 | 0.12 | 0.10 | 1.00 | -0.07 | 0.00 | 0.00 | -0.06 | 0.00 |
| is more concise | -0.99 | -0.44 | -0.13 | -0.07 | -0.07 | 1.00 | 0.02 | 0.02 | -0.01 | -0.11 |
| has a more avoidant tone | -0.02 | -0.03 | 0.03 | 0.06 | 0.00 | 0.02 | 1.00 | 0.80 | 0.06 | 0.15 |
| refuses to answer the question | -0.02 | -0.02 | 0.02 | 0.05 | 0.00 | 0.02 | 0.80 | 1.00 | 0.05 | 0.13 |
| ends with a follow-up question | 0.01 | -0.00 | 0.01 | -0.01 | -0.06 | -0.01 | 0.06 | 0.05 | 1.00 | 0.24 |
| is more polite | 0.12 | 0.21 | 0.17 | 0.11 | 0.00 | -0.11 | 0.15 | 0.13 | 0.24 | 1.00 |