Coverage for rdgai/prompts.py: 100.00%
75 statements
« prev ^ index » next coverage.py v7.6.4, created at 2025-01-03 01:37 +0000
1from langchain_core.messages import HumanMessage, AIMessage, SystemMessage
2from langchain.prompts import ChatPromptTemplate
3from .apparatus import Pair, Doc
6def select_spaced_elements(lst:list, k:int) -> list:
7 n = len(lst)
8 if k >= n:
9 return lst
10 if k == 1:
11 return [lst[0]]
12 if k == 2:
13 return [lst[0], lst[-1]]
15 # Always include the first and last element
16 indices = [0] + [int((n - 1) * i / (k - 1)) for i in range(1, k - 1)] + [n - 1]
18 return [lst[i] for i in indices]
21# def build_template(app:App, examples:int=10) -> ChatPromptTemplate:
22# doc = app.doc
24# readings = app.readings
26# system_message = f"You are an academic who is an expert in textual criticism in {language}."
28# human_message = (
29# f"I am analyzing textual variants in a {language} document.\n"
30# "I want to classify the types of changes made to readings as the text was copied.\n"
31# "If reading 1 were to change to reading 2, what type of change would that be?\n"
32# )
34# # Add the categories to the message
35# relation_categories = doc.relation_types.values()
36# human_message += f"\nHere are {len(relation_categories)} possible categories for the types of changes in the text:\n"
37# for category in relation_categories:
38# human_message += f"{category.str_with_description()}\n"
39# instance_strings = sorted(set(instance.reading_transition_str() for instance in category.pairs if not instance.rdgai_responsible()), key=len)
40# for instance_string in select_spaced_elements(instance_strings, examples):
41# human_message += f"\te.g. {instance_string}\n"
43# # Add the context to the message
44# human_message += f"\nI am variation unit is marked as {app.text_with_signs()} in this text:\n"
45# human_message += f"{app.text_in_context()}\n"
47# # Add the readings to the message
48# human_message += f"\nHere are the {len(readings)} readings at that variation unit. Remember them so you can analyze them later:\n"
49# for reading in readings:
50# reading_text = reading.text or "OMITTED"
51# human_message += f"{reading.n}: {reading_text}\n"
53# # Describe output format
54# human_message += (
55# f"\nYou need to output the classifications of the readings in the following format.\n"
56# # "If going reading 1 to reading 2 was the same category of change, you would output the following:\n"
57# # "reading_2_id <-> reading_1_id = category : justification\n"
59# "If reading 1 were to change to reading 2, you would output the following:\n"
60# "reading_1_id → reading_2_id = category : justification\n"
61# "You also should output classificaitons from reading 2 to reading 1. It could be the same category or a different one:\n"
62# "reading_2_id → reading_1_id = category : justification\n"
64# "For the justification, give a one sentence explanation of why the change is classified as that category.\n"
65# "Separate the result for each pair of readings with a newline and do not add extra new line characters.\n"
66# )
68# # Add the classified relation examples to the message
69# human_message += f"\nGiven the readings above, you can output the following pairs of relations:\n"
70# for i, reading1 in enumerate(readings):
71# for j in range(i + 1, len(readings)):
72# reading2 = readings[j]
73# human_message += f"{reading1.n} → {reading2.n}\n"
75# human_message += "\nIf the change between one of those pairs of readings does not fit one of the categories, then do not output anything for that pair.\n"
76# human_message += f"\nWhen you are finished, output 5 hyphens: '-----'.\n"
78# ai_message = "Certainly, classifications for combinations of the readings are:"
80# template = ChatPromptTemplate.from_messages(messages=[
81# SystemMessage(system_message),
82# HumanMessage(human_message),
83# AIMessage(ai_message),
84# ])
85# return template
88def build_system_message(doc:Doc) -> str:
89 return f"You are an academic who is an expert in textual criticism in {doc.language}."
92def build_preamble(doc:Doc, examples:int=10) -> str:
93 relation_categories = doc.relation_types.values()
94 human_message = (
95 f"I am analyzing textual variants in a document written in {doc.language}.\n"
96 f"There are {len(relation_categories)} possible categories for the types of changes in the text that a scribe could make to change one variant reading to another.\n"
97 "Here are the categories with their descriptions. Remember them because you will use this information later:\n"
98 )
99 for category in relation_categories:
100 human_message += f"{category.str_with_description()}\n"
102 human_message += "\n"
103 human_message += f"Here are examples of these categories The word 'OMIT' indicates the absence of text in this reading. If there is a justification describing the choice of category then it is added in square brackets []:\n"
104 for category in relation_categories:
105 representative_category_exampes = category.representative_examples(examples)
106 if len(representative_category_exampes) == 0:
107 continue
108 human_message += f"{category}:\n"
109 for pair in representative_category_exampes:
110 human_message += f"\te.g. {pair.reading_transition_str()}"
111 if pair.has_description():
112 human_message += f" [{pair.get_description()}]"
113 human_message += "\n"
115 human_message += "\n"
116 human_message += "I will give you a two variant readings. On the first line of your response, provide the correct category name for changing from the first reading to the second reading.\n"
117 human_message += "Do not provide any other text than the name of the category on the first line.\n"
118 human_message += "Then on a second line, give a justification of your decision according the definitions of the categories provided and similar examples.\n"
119 human_message += f"When you are finished, output 5 hyphens: '-----'.\n"
121 return human_message
124def build_template(pair:Pair, examples:int=10) -> ChatPromptTemplate:
125 app = pair.app
126 doc = app.doc
128 system_message = build_system_message(doc)
129 human_message = build_preamble(doc, examples)
131 human_message += f"\nThe variation unit you need to classify is marked as {app.text_with_signs(pair.active.text)} in this text:\n"
132 human_message += f"{app.text_in_context(pair.active.text)}\n"
134 active_reading_text = app.text_with_signs(str(pair.active))
135 passive_reading_text = app.text_with_signs(str(pair.passive))
136 human_message += f"\nWhat category would best describe a change from {active_reading_text} to {passive_reading_text}?\n"
138 relation_categories = doc.relation_types.values()
139 relation_categories_list = ", ".join(str(category) for category in relation_categories)
140 human_message += f"Respond with one of these categories: {relation_categories_list}\n"
141 human_message += f"On the second line, provide a justification for your decision."
143 ai_message = f"Certainly, the category for changing from {active_reading_text} to {passive_reading_text} is:"
145 template = ChatPromptTemplate.from_messages(messages=[
146 SystemMessage(system_message),
147 HumanMessage(human_message),
148 AIMessage(ai_message),
149 ])
150 return template
153def build_review_prompt(
154 doc:Doc,
155 correct_items:list["EvalItem"],
156 incorrect_items:list["EvalItem"],
157 examples:int=10,
158):
159 prompt_preamble = build_preamble(doc, examples=examples)
161 system_message = f"You are an expert prompt engineer with expertise in {doc.language}.\n"
163 human_message = "Please review the prompt based on results compared with the ground truth.\n"
164 human_message += f"Here is the prompt:\n```\n{prompt_preamble}```\n"
165 human_message += "----\nHere are the correctly classified items:\n"
166 for item in correct_items:
167 human_message += f" - { item.app_id }: {item.reading_transition_str} in `{item.text_in_context}` was correctly classified as {', '.join(item.predicted)} with this as the justification: {item.description}"
168 if item.ground_truth_description:
169 human_message += f" The justification for the ground truth {', '.join(item.ground_truth)} is: {item.ground_truth_description}"
170 human_message += "\n"
172 human_message += "----\nHere are the incorrectly classified items:\n"
173 for item in incorrect_items:
174 human_message += f" - { item.app_id }: {item.reading_transition_str} in `{item.text_in_context}` was incorrectly classified as {', '.join(item.predicted)} when it should have been {', '.join(item.ground_truth)}. This was the justification given: {item.description}"
175 if item.ground_truth_description:
176 human_message += f" The justification for the ground truth {', '.join(item.ground_truth)} is: {item.ground_truth_description}"
177 human_message += "\n"
179 human_message += "----\nPlease provide feedback on the prompt. Are the text be editing to improve the accuracy of the results? Are there examples in the test set where the ground truth label is incorrect? If so, then list all the problematic cases with their locations.\n"
181 template = ChatPromptTemplate.from_messages(messages=[
182 SystemMessage(system_message),
183 HumanMessage(human_message),
184 ])
186 return template