解密prompt系列35. 标准化Prompt进行时！ DSPy论文串烧和代码示例

TIPS = {
        "none": "",
        "creative": "Don't be afraid to be creative when creating the new instruction!",
        "simple": "Keep the instruction clear and concise.",
        "description": "Make sure your instruction is very informative and descriptive.",
        "high_stakes": "The instruction should include a high stakes scenario in which the LM must solve the task!",
        "persona": 'Include a persona that is relevant to the task in the instruction (ie. "You are a ...")',
    }
class GenerateSingleModuleInstruction(dspy.Signature):
    (
        """Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task."""
    )
    if use_dataset_summary:
        dataset_description = dspy.InputField(
            desc="A description of the dataset that we are using.",
            prefix="DATASET SUMMARY:",
        )
    if program_aware:
        program_code = dspy.InputField(
            format=str,
            desc="Language model program designed to solve a particular task.",
            prefix="PROGRAM CODE:",
        )
        program_description = dspy.InputField(
            desc="Summary of the task the program is designed to solve, and how it goes about solving it.",
            prefix="PROGRAM DESCRIPTION:",
        )
        module = dspy.InputField(
            desc="The module to create an instruction for.", prefix="MODULE:",
        )
    task_demos = dspy.InputField(
        format=str,
        desc="Example inputs/outputs of our module.",
        prefix="TASK DEMO(S):",
    )
    if use_instruct_history:
        previous_instructions = dspy.InputField(
            format=str,
            desc="Previous instructions we've attempted, along with their associated scores.",
            prefix="PREVIOUS INSTRUCTIONS:",
        )
    basic_instruction = dspy.InputField(
        format=str, desc="Basic instruction.", prefix="BASIC INSTRUCTION:",
    )
    if use_tip:
        tip = dspy.InputField(
            format=str,
            desc="A suggestion for how to go about generating the new instruction.",
            prefix="TIP:",
        )
    proposed_instruction = dspy.OutputField(
        desc="Propose an instruction that will be used to prompt a Language Model to perform this task.",
        prefix="PROPOSED INSTRUCTION:",
    )

import dspy
model = dspy.AzureOpenAI(**kwargs)
dspy.settings.configure(lm=model)

class SingleChoiceQA(dspy.Signature):
    """单项选择题，给定题目和ABCD四个选项，输出正确的选项"""
    question = dspy.InputField(desc='问题和选项')
    answer = dspy.OutputField(desc="[ABCD]之一的正确选项")

def format_example(line):
    input = line['question']
    for choice in ['A', 'B', 'C', 'D']:
        input += f'\n{choice}. {line[f"{choice}"]}'
    output = line["answer"]
    example = dspy.Example(question=input, answer=output).with_inputs('question')
    return example

class SQA(dspy.Module):
    def __init__(self):
        self.generate_answer = dspy.Predict(SingleChoiceQA)

    def forward(self, question):
        return self.generate_answer(question=question)
singlechoice_qa = SQA()
pred = singlechoice_qa(question=examples[0]['question'])
print('以下为DSPy生成Prompt')
print(model.inspect_history(n=1))

class SQACOT(dspy.Module):
    def __init__(self):
        self.generate_answer = dspy.ChainOfThought(SingleChoiceQA)

    def forward(self, question):
        return self.generate_answer(question=question)
singlechoice_qa_cot = SQACOT()

from dspy.evaluate.evaluate import Evaluate
def choice_match(example, pred, trace=None):
    def extract_choice(gen_ans):
        m = re.findall(r'[ABCD]', gen_ans, re.M)
        if len(m) >= 1:
            answer = m[0]
            return answer
        return random.choice('ABCD')

    return extract_choice(pred.answer) == example.answer

evaluate_on_qa = Evaluate(devset=test, num_threads=1,
                          display_progress=True, display_table=True)
output1 = evaluate_on_qa(singlechoice_qa, metric=choice_match,return_outputs=True,return_all_scores=True)

output2 = evaluate_on_qa(singlechoice_qa_cot, metric=choice_match,return_outputs=True,return_all_scores=True)

teleprompter = BootstrapFewShot(metric=choice_match,
                                max_bootstrapped_demos=4,
                                max_labeled_demos=16)
compiled_qa = teleprompter.compile(singlechoice_qa_cot, trainset=train)

score3, results3, output3, df3 = evaluate_on_qa(compiled_qa, metric=choice_match,
                                                return_outputs=True,
                                                return_all_scores=True)

COPRO_teleprompter = COPRO(prompt_model=model ,
                          metric=choice_match,
                          breadth=5,
                          depth=3,
                          init_temperature=0.7,
                          track_stats=True)
kwargs = dict(num_threads=1, display_progress=True, display_table=5)
COPRO_compiled_qa = COPRO_teleprompter.compile(singlechoice_qa_cot,
                                                trainset=train,
                                                eval_kwargs=kwargs)
print(model.inspect_history(n=1))