From aa21d0d9d3395eb4b21be0284c2fab7b5cc944ea Mon Sep 17 00:00:00 2001 From: Matteo Villosio Date: Sat, 16 Dec 2023 09:54:38 +0100 Subject: [PATCH 1/5] Changed standard final_answer_pattern in base.py to avoid stopping match with newline --- haystack/agents/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/haystack/agents/base.py b/haystack/agents/base.py index f8b2504128..539fc73eaa 100644 --- a/haystack/agents/base.py +++ b/haystack/agents/base.py @@ -214,7 +214,7 @@ def __init__( memory: Optional[Memory] = None, prompt_parameters_resolver: Optional[Callable] = None, max_steps: int = 8, - final_answer_pattern: str = r"Final Answer\s*:\s*(.*)", + final_answer_pattern: str = r"(?s)Final Answer\s*:\s*(.*)", streaming: bool = True, ): """ From dfaf3405084f04712c1e323fcd10c019e3dbe393 Mon Sep 17 00:00:00 2001 From: Matteo Villosio Date: Mon, 18 Dec 2023 10:08:09 +0100 Subject: [PATCH 2/5] added testing on multiline --- test/agents/test_agent.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/agents/test_agent.py b/test/agents/test_agent.py index df7511bb6a..e2a1c03c6a 100644 --- a/test/agents/test_agent.py +++ b/test/agents/test_agent.py @@ -149,6 +149,7 @@ def test_extract_final_answer(): "Final Answer:42", "Final Answer: ", "Final Answer: The answer is 99 ", + "Final Answer: 42 should be the answer\n\nBut it's not the only one\n\n\n\n the answer is 1948", ] expected_answers = [ "Florida", @@ -159,6 +160,7 @@ def test_extract_final_answer(): "42", "", "The answer is 99", + "42 should be the answer\n\nBut it's not the only one\n\n\n\n the answer is 1948" ] for example, expected_answer in zip(match_examples, expected_answers): From e17e90eb4eb4b37a5d6a3ec0d2ea4dc17ff8451e Mon Sep 17 00:00:00 2001 From: Matteo Villosio Date: Mon, 18 Dec 2023 10:17:59 +0100 Subject: [PATCH 3/5] added testing on final_answer_pattern for multiline matching --- test/agents/test_agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/agents/test_agent.py b/test/agents/test_agent.py index e2a1c03c6a..34302a29e6 100644 --- a/test/agents/test_agent.py +++ b/test/agents/test_agent.py @@ -149,7 +149,7 @@ def test_extract_final_answer(): "Final Answer:42", "Final Answer: ", "Final Answer: The answer is 99 ", - "Final Answer: 42 should be the answer\n\nBut it's not the only one\n\n\n\n the answer is 1948", + "Final Answer:42 should be the answer\n\nBut it's not the only one\n\n\n\n the answer is 1948", ] expected_answers = [ "Florida", @@ -164,7 +164,7 @@ def test_extract_final_answer(): ] for example, expected_answer in zip(match_examples, expected_answers): - agent_step = AgentStep(prompt_node_response=example, final_answer_pattern=r"Final Answer\s*:\s*(.*)") + agent_step = AgentStep(prompt_node_response=example, final_answer_pattern=r"(?s)Final Answer\s*:\s*(.*)") final_answer = agent_step.final_answer(query="irrelevant") assert final_answer["answers"][0].answer == expected_answer From 96f8c24b383e79926fc6a83628265c90e8b5fb1b Mon Sep 17 00:00:00 2001 From: Matteo Villosio Date: Mon, 18 Dec 2023 10:48:18 +0100 Subject: [PATCH 4/5] reformatting --- test/agents/test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/agents/test_agent.py b/test/agents/test_agent.py index 34302a29e6..73af75d146 100644 --- a/test/agents/test_agent.py +++ b/test/agents/test_agent.py @@ -160,7 +160,7 @@ def test_extract_final_answer(): "42", "", "The answer is 99", - "42 should be the answer\n\nBut it's not the only one\n\n\n\n the answer is 1948" + "42 should be the answer\n\nBut it's not the only one\n\n\n\n the answer is 1948", ] for example, expected_answer in zip(match_examples, expected_answers): From 276308d4bb5f4e8679b0ac7c76624beb2de48d10 Mon Sep 17 00:00:00 2001 From: Matteo Villosio Date: Mon, 18 Dec 2023 10:59:51 +0100 Subject: [PATCH 5/5] added release note --- .../improve-agent-final-answer-matching-21f5710035a498d8.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 releasenotes/notes/improve-agent-final-answer-matching-21f5710035a498d8.yaml diff --git a/releasenotes/notes/improve-agent-final-answer-matching-21f5710035a498d8.yaml b/releasenotes/notes/improve-agent-final-answer-matching-21f5710035a498d8.yaml new file mode 100644 index 0000000000..43e2fe717e --- /dev/null +++ b/releasenotes/notes/improve-agent-final-answer-matching-21f5710035a498d8.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fixed the bug that caused the agent to discard part of final answers if they were distributed across multiple lines.