forked from agent-lab/smolagents
08319f2f77
Co-authored-by: Albert Villanova del Moral <8515462+albertvillanova@users.noreply.github.com>
457 lines
13 KiB
Text
457 lines
13 KiB
Text
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"!pip install plotly kaleido datasets nbformat -U -q"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import os\n",
|
|
"\n",
|
|
"import datasets\n",
|
|
"import pandas as pd\n",
|
|
"from dotenv import load_dotenv\n",
|
|
"from huggingface_hub import login\n",
|
|
"\n",
|
|
"\n",
|
|
"load_dotenv(override=True)\n",
|
|
"login(os.getenv(\"HF_TOKEN\"))\n",
|
|
"\n",
|
|
"pd.set_option(\"max_colwidth\", None)\n",
|
|
"\n",
|
|
"OUTPUT_DIR = \"output\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"eval_ds = datasets.load_dataset(\"gaia-benchmark/GAIA\", \"2023_all\")[\"validation\"]\n",
|
|
"eval_ds = eval_ds.rename_columns({\"Question\": \"question\", \"Final answer\": \"true_answer\", \"Level\": \"task\"})\n",
|
|
"eval_df = pd.DataFrame(eval_ds)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# 1. Load all results"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 88,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import glob\n",
|
|
"\n",
|
|
"\n",
|
|
"results = []\n",
|
|
"for f in glob.glob(f\"{OUTPUT_DIR}/validation/*.jsonl\"):\n",
|
|
" df = pd.read_json(f, lines=True)\n",
|
|
" df[\"agent_name\"] = f.split(\"/\")[-1].split(\".\")[0]\n",
|
|
" results.append(df)\n",
|
|
"\n",
|
|
"result_df = pd.concat(results)\n",
|
|
"result_df[\"prediction\"] = result_df[\"prediction\"].fillna(\"No prediction\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import re\n",
|
|
"from collections import Counter\n",
|
|
"\n",
|
|
"from scripts.gaia_scorer import check_close_call, question_scorer\n",
|
|
"\n",
|
|
"\n",
|
|
"result_df[\"is_correct\"] = result_df.apply(lambda x: question_scorer(x[\"prediction\"], x[\"true_answer\"]), axis=1)\n",
|
|
"result_df[\"is_near_correct\"] = result_df.apply(\n",
|
|
" lambda x: check_close_call(x[\"prediction\"], x[\"true_answer\"], x[\"is_correct\"]),\n",
|
|
" axis=1,\n",
|
|
")\n",
|
|
"\n",
|
|
"result_df[\"count_steps\"] = result_df[\"intermediate_steps\"].apply(len)\n",
|
|
"\n",
|
|
"\n",
|
|
"def find_attachment(question):\n",
|
|
" matches = eval_df.loc[eval_df[\"question\"].apply(lambda x: x in question), \"file_name\"]\n",
|
|
"\n",
|
|
" if len(matches) == 0:\n",
|
|
" return \"Not found\"\n",
|
|
" file_path = matches.values[0]\n",
|
|
"\n",
|
|
" if isinstance(file_path, str) and len(file_path) > 0:\n",
|
|
" return file_path.split(\".\")[-1]\n",
|
|
" else:\n",
|
|
" return \"None\"\n",
|
|
"\n",
|
|
"\n",
|
|
"result_df[\"attachment_type\"] = result_df[\"question\"].apply(find_attachment)\n",
|
|
"\n",
|
|
"\n",
|
|
"def extract_tool_calls(code):\n",
|
|
" regex = r\"\\b(\\w+)\\(\"\n",
|
|
" function_calls = [el for el in re.findall(regex, code) if el.islower()]\n",
|
|
"\n",
|
|
" function_call_counter = Counter(function_calls)\n",
|
|
" return function_call_counter\n",
|
|
"\n",
|
|
"\n",
|
|
"def sum_tool_calls(steps):\n",
|
|
" total_count = Counter()\n",
|
|
" for step in steps:\n",
|
|
" if \"llm_output\" in step:\n",
|
|
" total_count += extract_tool_calls(step[\"llm_output\"])\n",
|
|
"\n",
|
|
" return total_count\n",
|
|
"\n",
|
|
"\n",
|
|
"def get_durations(row):\n",
|
|
" # start_datetime = datetime.strptime(row['start_time'], \"%Y-%m-%d %H:%M:%S\")\n",
|
|
" # end_datetime = datetime.strptime(row['end_time'], \"%Y-%m-%d %H:%M:%S\")\n",
|
|
"\n",
|
|
" duration_timedelta = row[\"end_time\"] - row[\"start_time\"]\n",
|
|
" return int(duration_timedelta.total_seconds())\n",
|
|
"\n",
|
|
"\n",
|
|
"result_df[\"duration\"] = result_df.apply(get_durations, axis=1)\n",
|
|
"# result_df[\"tool_calls\"] = result_df[\"intermediate_steps\"].apply(sum_tool_calls)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"result_df[\"agent_name\"].value_counts()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# 2. Inspect specific runs"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"sel_df = result_df\n",
|
|
"# sel_df = sel_df.loc[\n",
|
|
"# (result_df[\"agent_name\"].isin(list_versions))\n",
|
|
"# ]\n",
|
|
"sel_df = sel_df.reset_index(drop=True)\n",
|
|
"display(sel_df[\"agent_name\"].value_counts())\n",
|
|
"sel_df = sel_df.drop_duplicates(subset=[\"agent_name\", \"question\"])\n",
|
|
"display(sel_df.groupby(\"agent_name\")[[\"task\"]].value_counts())\n",
|
|
"print(\"Total length:\", len(sel_df), \"- is complete:\", len(sel_df) == 165)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"display(\"Average score:\", sel_df.groupby(\"agent_name\")[[\"is_correct\"]].mean().round(3))\n",
|
|
"display(\n",
|
|
" sel_df.groupby([\"agent_name\", \"task\"])[[\"is_correct\", \"is_near_correct\", \"count_steps\", \"question\", \"duration\"]]\n",
|
|
" .agg(\n",
|
|
" {\n",
|
|
" \"is_correct\": \"mean\",\n",
|
|
" \"is_near_correct\": \"mean\",\n",
|
|
" \"count_steps\": \"mean\",\n",
|
|
" \"question\": \"count\",\n",
|
|
" \"duration\": \"mean\",\n",
|
|
" }\n",
|
|
" )\n",
|
|
" .rename(columns={\"question\": \"count\"})\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import plotly.express as px\n",
|
|
"\n",
|
|
"\n",
|
|
"cumulative_df = (\n",
|
|
" (\n",
|
|
" sel_df.groupby(\"agent_name\")[[\"is_correct\", \"is_near_correct\"]]\n",
|
|
" .expanding(min_periods=1, axis=0, method=\"single\")\n",
|
|
" .agg({\"is_correct\": \"mean\", \"is_near_correct\": \"count\"})\n",
|
|
" .reset_index()\n",
|
|
" )\n",
|
|
" .copy()\n",
|
|
" .rename(columns={\"is_near_correct\": \"index\"})\n",
|
|
")\n",
|
|
"cumulative_df[\"index\"] = cumulative_df[\"index\"].astype(int) - 1\n",
|
|
"\n",
|
|
"\n",
|
|
"def find_question(row):\n",
|
|
" try:\n",
|
|
" res = sel_df.loc[sel_df[\"agent_name\"] == row[\"agent_name\"], \"question\"].iloc[row[\"index\"]][:50]\n",
|
|
" return res\n",
|
|
" except Exception:\n",
|
|
" return \"\"\n",
|
|
"\n",
|
|
"\n",
|
|
"cumulative_df[\"question\"] = cumulative_df.apply(find_question, axis=1)\n",
|
|
"\n",
|
|
"px.line(\n",
|
|
" cumulative_df,\n",
|
|
" color=\"agent_name\",\n",
|
|
" x=\"index\",\n",
|
|
" y=\"is_correct\",\n",
|
|
" hover_data=\"question\",\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# 3. Dive deeper into one run"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"sel_df = result_df.loc[result_df[\"agent_name\"] == \"o1\"]\n",
|
|
"print(len(sel_df))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Count errors"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import numpy as np\n",
|
|
"\n",
|
|
"\n",
|
|
"error_types = [\n",
|
|
" \"AgentParsingError\",\n",
|
|
" \"AgentExecutionError\",\n",
|
|
" \"AgentMaxIterationsError\",\n",
|
|
" \"AgentGenerationError\",\n",
|
|
"]\n",
|
|
"sel_df[error_types] = 0\n",
|
|
"sel_df[\"Count steps\"] = np.nan\n",
|
|
"\n",
|
|
"\n",
|
|
"def count_errors(row):\n",
|
|
" if isinstance(row[\"intermediate_steps\"], list):\n",
|
|
" row[\"Count steps\"] = len(row[\"intermediate_steps\"])\n",
|
|
" for step in row[\"intermediate_steps\"]:\n",
|
|
" if isinstance(step, dict) and \"error\" in step:\n",
|
|
" try:\n",
|
|
" row[str(step[\"error\"][\"error_type\"])] += 1\n",
|
|
" except Exception:\n",
|
|
" pass\n",
|
|
" return row\n",
|
|
"\n",
|
|
"\n",
|
|
"sel_df = sel_df.apply(count_errors, axis=1)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import plotly.express as px\n",
|
|
"\n",
|
|
"\n",
|
|
"aggregate_errors = (\n",
|
|
" sel_df.groupby([\"is_correct\"])[error_types + [\"Count steps\"]].mean().reset_index().melt(id_vars=[\"is_correct\"])\n",
|
|
")\n",
|
|
"\n",
|
|
"fig = px.bar(\n",
|
|
" aggregate_errors,\n",
|
|
" y=\"value\",\n",
|
|
" x=\"variable\",\n",
|
|
" color=\"is_correct\",\n",
|
|
" labels={\n",
|
|
" \"agent_name\": \"<b>Model</b>\",\n",
|
|
" \"task\": \"<b>Level</b>\",\n",
|
|
" \"aggregate_score\": \"<b>Performance</b>\",\n",
|
|
" \"value\": \"<b>Average count</b>\",\n",
|
|
" \"eval_score_GPT4\": \"<b>Score</b>\",\n",
|
|
" },\n",
|
|
")\n",
|
|
"fig.update_layout(\n",
|
|
" height=500,\n",
|
|
" width=800,\n",
|
|
" barmode=\"group\",\n",
|
|
" bargroupgap=0.0,\n",
|
|
")\n",
|
|
"fig.update_traces(textposition=\"outside\")\n",
|
|
"fig.write_image(\"aggregate_errors.png\", scale=3)\n",
|
|
"fig.show()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Inspect result by file extension type"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"display(\n",
|
|
" result_df.groupby([\"attachment_type\"])[[\"is_correct\", \"count_steps\", \"question\"]].agg(\n",
|
|
" {\"is_correct\": \"mean\", \"count_steps\": \"mean\", \"question\": \"count\"}\n",
|
|
" )\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# 4. Ensembling methods"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"counts = result_df[\"agent_name\"].value_counts()\n",
|
|
"long_series = result_df.loc[result_df[\"agent_name\"].isin(counts[counts > 140].index)]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def majority_vote(df):\n",
|
|
" df = df[(df[\"prediction\"] != \"Unable to determine\") & (~df[\"prediction\"].isna()) & (df[\"prediction\"] != \"None\")]\n",
|
|
"\n",
|
|
" answer_modes = df.groupby(\"question\")[\"prediction\"].agg(lambda x: x.mode()[0]).reset_index()\n",
|
|
" first_occurrences = (\n",
|
|
" df.groupby([\"question\", \"prediction\"]).agg({\"task\": \"first\", \"is_correct\": \"first\"}).reset_index()\n",
|
|
" )\n",
|
|
" result = answer_modes.merge(first_occurrences, on=[\"question\", \"prediction\"], how=\"left\")\n",
|
|
"\n",
|
|
" return result\n",
|
|
"\n",
|
|
"\n",
|
|
"def oracle(df):\n",
|
|
" def get_first_correct_or_first_wrong(group):\n",
|
|
" correct_answers = group[group[\"is_correct\"]]\n",
|
|
" if len(correct_answers) > 0:\n",
|
|
" return correct_answers.iloc[0]\n",
|
|
" return group.iloc[0]\n",
|
|
"\n",
|
|
" result = df.groupby(\"question\").apply(get_first_correct_or_first_wrong)\n",
|
|
"\n",
|
|
" return result.reset_index(drop=True)\n",
|
|
"\n",
|
|
"\n",
|
|
"display((long_series.groupby(\"agent_name\")[\"is_correct\"].mean() * 100).round(2))\n",
|
|
"print(f\"Majority score: {majority_vote(long_series)['is_correct'].mean() * 100:.2f}\")\n",
|
|
"print(f\"Oracle score: {oracle(long_series)['is_correct'].mean() * 100:.2f}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### Submit"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"agent_run = \"code_o1_04_february_submission5.jsonl\"\n",
|
|
"df = pd.read_json(f\"output/validation/{agent_run}\", lines=True)\n",
|
|
"df = df[[\"task_id\", \"prediction\", \"intermediate_steps\"]]\n",
|
|
"df = df.rename(columns={\"prediction\": \"model_answer\", \"intermediate_steps\": \"reasoning_trace\"})"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df.to_json(\"submission.jsonl\", orient=\"records\", lines=True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "agents",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.12.0"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|