Lesson 30: Evaluating the Writing Workflow

class BrownBaseMetric(base_metric.BaseMetric, Generic[FewShotExamplesT, StructuredOutputTypeT], abc.ABC):
        """Abstract base class for Brown evaluation metrics that use LLMs for structured scoring."""
    
        def __init__(
            self,
            model: SupportedModels,
            name: str,
            structured_output_type: type[StructuredOutputTypeT],
            few_shot_examples: FewShotExamplesT,
            model_config: ModelConfig,
            track: bool = True,
            project_name: str | None = None,
    ) -> None:
            super().__init__(
                name=name,
                track=track,
                project_name=project_name,
            )
    
            self.model = model
            self.structured_output_type = structured_output_type
            self.model_config = model_config
            self.few_shot_examples = few_shot_examples
    
            if self.model == SupportedModels.FAKE_MODEL:
                assert self.model_config and self.model_config.mocked_response is not None

def init_model(self) -> Runnable:
            """Initialize the language model with structured output capabilities."""
            model_instance = get_model(self.model, self.model_config)
            model_instance = model_instance.with_structured_output(self.structured_output_type)
            return model_instance
    
        def score(self, *args: Any, **kwargs: Any) -> score_result.ScoreResult | list[score_result.ScoreResult]:
            """Calculate evaluation scores synchronously by wrapping the async implementation."""
            return a.asyncio_run(self.ascore(*args, **kwargs))
    
        @abc.abstractmethod
        async def ascore(self, *args: Any, **kwargs: Any) -> score_result.ScoreResult | list[score_result.ScoreResult]:
            """Abstract async method for implementing metric-specific evaluation logic."""
            pass

class FollowsGTMetricLLMJudge(BrownBaseMetric):
        """A metric that evaluates the quality of article content across multiple sections and dimensions."""
    
        def __init__(
            self,
            model: SupportedModels = SupportedModels.GOOGLE_GEMINI_25_FLASH,
            name: str = "follows_gt",
            model_config: ModelConfig | None = None,
            track: bool = True,
            project_name: str | None = None,
    ) -> None:
            model_config = model_config or ModelConfig(
                temperature=0.0, thinking_budget=1024 * 4, include_thoughts=False, max_retries=3
            )
            super().__init__(
                model=model,
                name=name,
                structured_output_type=FollowsGTArticleScores,
                few_shot_examples=prompts.DEFAULT_FEW_SHOT_EXAMPLES,
                model_config=model_config,
                track=track,
                project_name=project_name,
            )

async def ascore(
            self,
            output: str,
            expected_output: str,
            **ignored_kwargs: Any,
    ) -> score_result.ScoreResult | list[score_result.ScoreResult]:
            # Initialize the model client at the function level to avoid coroutine reuse issues
            model_client = self.init_model()
    
            llm_query = prompts.get_eval_prompt(
                output=output,
                expected_output=expected_output,
                few_shot_examples=self.few_shot_examples,
            )
            article_response = cast(
                FollowsGTArticleScores,
                await model_client.ainvoke([{"role": "user", "content": llm_query}]),
            )
    
            if not article_response:
                raise ValueError("Model failed to return a structured response.")
    
            return article_response.to_score_result(self.name)

class FollowsGTArticleScores(pydantic.BaseModel):
        """Article-level scores for the FollowsGT evaluation metric."""
    
        sections: list[FollowsGTSectionScores]
    
        def to_score_result(self, prefix: str) -> list[score_result.ScoreResult]:
            """Convert the evaluation results to ScoreResult objects with dimension-wise scoring."""
            return aggregate_section_scores_to_results(self.sections, prefix)

class FollowsGTSectionScores(pydantic.BaseModel):
        """Section evaluation with scores across all FollowsGT dimensions."""
    
        title: str = pydantic.Field(description="The title of the section being evaluated.")
        scores: FollowsGTCriterionScores = pydantic.Field(description="The scores of the section.")
    
    class FollowsGTCriterionScores(CriteriaScores):
        """Represents scores for all three evaluation dimensions of a section."""
    
        content: CriterionScore
        flow: CriterionScore
        structure: CriterionScore

class CriterionScore(pydantic.BaseModel):
        """Base model for a single score representing a specific evaluation dimension."""
    
        score: Annotated[int, Ge(0), Le(1)] = pydantic.Field(description="Binary score of the section.")
        reason: str = pydantic.Field(description="The reason for the given score.")

class CriteriaScores(pydantic.BaseModel):
        """Abstract base class for scores across multiple evaluation dimensions."""
    
        def to_context(self) -> str:
            """Convert the scores to a formatted XML string for use as context in prompts."""
            scores_fields = self.__class__.model_fields
            scores_xml = ""
            for field_name in scores_fields.keys():
                field_score = getattr(self, field_name)
                scores_xml += f"""    <{field_name}>
            <score>{field_score.score}</score>
            <reason>{field_score.reason}</reason>
        </{field_name}>
    """
            return scores_xml

def aggregate_section_scores_to_results(
        section_scores: list,
        prefix: str,
    ) -> list[score_result.ScoreResult]:
        """Convert section-level evaluation results to aggregated ScoreResult objects."""
        if not section_scores:
            return []
    
        # Automatically infer dimensions from the first section's scores class
        scores_class = type(section_scores[0].scores)
        scores_fields = scores_class.model_fields
        aggregated_scores: dict[str, dict[str, list[int] | str]] = {
            field_name: {"scores": [], "reason": ""}
            for field_name in scores_fields.keys()
        }

for section in section_scores:
            for dimension in aggregated_scores.keys():
                dimension_score = getattr(section.scores, dimension)
                aggregated_scores[dimension]["scores"].append(dimension_score.score)
                aggregated_scores[dimension]["reason"] += f"{section.title}:\n"
                aggregated_scores[dimension]["reason"] += f"**{dimension_score.score}:** {dimension_score.reason}\n\n"

results: list[score_result.ScoreResult] = []
        for dimension, scores_data in aggregated_scores.items():
            scores_list = scores_data["scores"]
            aggregated_score = CriterionAggregatedScore(
                name=f"{prefix}_{dimension}",
                score=sum(scores_list) / len(scores_list),
                reason=str(scores_data["reason"]),
            )
            results.append(aggregated_score.to_score_result())
    
        return results

class CriterionAggregatedScore(pydantic.BaseModel):
        """Represents an aggregated score for a specific evaluation dimension."""
    
        name: str
        score: float
        reason: str
    
        def to_score_result(self) -> score_result.ScoreResult:
            return score_result.ScoreResult(name=self.name, value=self.score, reason=self.reason)

class FollowsGTMetricFewShotExample(BaseExample):
        """Represents a single example for the follows_gt evaluation."""
    
        output: str
        expected_output: str
        scores: FollowsGTArticleScores
    
        @classmethod
        def from_markdown(
            cls, output_file: Path, expected_output_file: Path, scores: FollowsGTArticleScores
    ) -> "FollowsGTMetricFewShotExample":
            output = output_file.read_text()
            expected_output = expected_output_file.read_text()
            return cls(output=output, expected_output=expected_output, scores=scores)
    
        def to_context(self) -> str:
            return f"""
    <output>
    {self.output}
    </output>
    <expected_output>
    {self.expected_output}
    </expected_output>
    {self.scores.to_context()}
    """

class BaseExample(pydantic.BaseModel, Generic[ExampleT]):
        """Base class for examples used in evaluation metrics."""
    
        @abc.abstractmethod
        def to_context(self) -> str:
            """Convert the example to a formatted string for use as context in prompts."""
            pass
    
    class BaseFewShotExamples(pydantic.BaseModel, Generic[ExampleT]):
        """Base class for few-shot examples collections."""
    
        examples: list[ExampleT]
    
        def to_context(self) -> str:
            examples = "\n\n".join(
                [f"<example_{i + 1}>\n\t{example.to_context()}\n</example_{i + 1}>\n"
                 for i, example in enumerate(self.examples)]
            )
            return examples

SYSTEM_PROMPT = """
    You are an expert in Natural Language Processing (NLP) evaluation metrics, specifically trained to
    assess answer quality in responses provided by LLMs.
    
    Your task is to evaluate the quality of a generated article by another LLM relative to
    an expected article output across multiple criteria: content, flow, structure, and mechanics.
    ...
    """

"""
    ...
    ## INSTRUCTIONS
    
    1. You must analyze the given expected article (<expected_output>) and generated article (<generated_output>)
    to determine the most relevant evaluation.
    2. Since the generated output is an answer from another LLM, you must use the expected output as the reference
    standard to compare and evaluate the quality of the generated output.
    3. Both the generated and expected outputs are in Markdown format.
    4. Instead of comparing the outputs as a whole, you will divide the outputs into sections and compare each section
    individually.
    5. You will always use the expected output as the reference point to extract the sections of interest during the
    evaluation. If there is no perfect match between the expected and generated section names, first try to infer
    the corresponding section based on the similarity of section names and their respective content. If you conclude that
    the expected output contains a section that the generated output lacks, you will assign a score of 0 to the missing
    section in the generated output.
    6. Sections are divided by H2 headers, marked as "##" in Markdown. You will use these headers as
    separators. Anything between two H2 headers constitutes a section. The only valid exception to this rule is the first
    section, the introduction, which sometimes appears between the title and the first H2 header. You will never include
    the title or subtitle as part of the first section.
    7. When comparing each individual section of the expected output to the generated output, you will assign a binary
    score for multiple criteria: 0 or 1, where 0 indicates a non-match and 1 indicates a perfect match. Each criterion
    is completely independent of the others, meaning that a score of 0 in one criterion does not affect the score of
    another criterion.
    8. You must compute binary scores for each section based on the following criteria:
    ...
    """

"""
    ...
    	1. **Content:** Evaluate whether the generated section covers the same content as the expected section:
        - Focus only on evaluating that the substance of the content is the same between the expected and generated section. By content, we
        mean core subjects, topics, research, ideas, key points or arguments. For example, if both sections discuss the
        fundamentals of RAG, it's valid. But, if the expected section discusses advanced RAG topics, while the generated section
        discusses basic RAG topics, it's invalid.
        - In this criterion, we are not interested in the order, structure, layout, or any other aspect related to the flow of
        ideas, structure or mechanics. For example, if there are missing or additional ideas discussed it's still valid, as
        long as the substance of the content between the expected and generated section is the same.
    ...
    """

"""
    ...
    	2. **Flow:** Evaluate whether the generated section follows the same order of ideas as the expected section, such as
    	the flow of:
    	    - Main ideas covered starting with the beginning, until the end of the section. With special emphasis on the beginning and end of the
    	    section as they reflect the transition between the previous and next sections
    	    - Internal transitions between the main points within the section. We expect a smooth flow of ideas,
    	    without any abrupt jumps or breaks.
    	    - Placement of notes, images, tables, code blocks, or any other media elements within the generated section,
    	    relative to the expected section.
    	    - We don't expect a perfect one on one match between the paragraphs and sentences between the expected and generated section.
    	    However, we expect the same ideas and concepts to be discussed in the same way, order, and storyline.
    	    - Assign a score of 0 if anything is missing from the generated section relative to the expected section, such as missing
    	    topics, ideas, or media.
    	    - Assign a score of 0 if anything is additionally added to the generated section relative to the expected section, such
    	    as additional topics, ideas or media elements.
    	    - Accepted differences between the expected and generated section:
    	        - Mismatching media numbering is accepted. For example, if in the expected section we have a figure with the number 3 and
    	        in the generated section we have a figure with the number 4, it's valid. It will be invalid, only if the figure would be
    	        missing altogether.
    	        - Mismatching or missing emojis. For example, if the excepted section has a 💡 emoji, while the generated section has
    	        a 🔑 emoji, it's valid. Also, if the emoji is missing altogether from the generated section, it's valid.
    	        - Mismatching source reference numbers. For example, if the expected section referes a source with the number 3,
    	        while the generated section referes a source with the number 7, it's valid. It will be invalid, only if the generated
    	        section misses the source altogether.
    	        - Different placement of the source in the generated section. For example, if the expected section has the source
    	        at the end of a sentence within the paragraph, while the generated section has it at the end of the paragraph, it's valid.
    	        It will be invalid, only if the generated section would be missing altogether.
    	        - Mismatching number of source references. For example, if the expected section has 3 source references, while the
    	        generated section has 2 source references, it's valid. It will be invalid, only if the generated section would have
    	        misses the references altogether. For example if the expected section has 3 source references, while the generated
    	        section has 0 source references, it's invalid.
    	        - Having reference numbers in the generated section, while having none in the expected section. For example, if the
    	        expected section has 0 reference numbers, while the generated section has 3 reference number, it's valid. It will be
    	        invalid, only the other way around, where the expected section has 3 reference numbers, while the generated section has 0.
    ...
    """

"""
    ...
    	3. **Structure:** Evaluate whether the generated section follows the same structure as the expected section. By
    	structure, we mean:
    	    - H3/H4/H5/H6 sub-heading structure and formatting
    	    - Mismatches in headers formatting and presence. For example, if the expected section doesn't has a header,
    	    while the generated section has one, it's invalid. It's valid only if there is a one on one match between the headers
    	    formatting and presence.
    	    - Use of bulleted lists, numbered lists, callouts, notes, or other layout elements
    	    - Division of the section when guiding readers through code blocks or diagrams
    	    - Formatting of notes and code blocks
    	    - Use of bolding, italicizing, quotes, backticks, or other formatting elements
    	    - Formatting of citation references across sentences
    	    - Formatting of images, tables, and Mermaid diagrams and their corresponding citations. If they are missing from
    	    the generated section, we consider it valid for this criterion, as we are interested ONLY in formatting, which we
    	    cannot verify when elements are absent. For this criterion, missing elements from the generated sections are
    	    considered valid. They are invalid only if present in both sections but formatted differently.
    	    - Number formatting conventions
    ...
    """

"""
    ...
    9. Along with the binary scores, you will provide a brief and concise explanation containing the reasoning behind
    the score for each criterion. The score will be used to debug and monitor the evaluation process. Therefore, it is
    important to provide thorough reasoning for the score. Since we provide binary scores, the reasoning should always
    contain what is good and what is problematic about the generated section, regardless of the score. For example, if the
    score is 0, the reasoning should also contain what is good about the generated section, such as "both sections
    follow the same flow of ideas," and what is problematic, such as "the generated section contains an additional
    paragraph on AI Evals that is not present in the expected section."
    10. Important rules when comparing the content of sections:
        - Focus on substance, not superficial formatting differences
        - When comparing **media**, you only care about the placement of the media, not the content of the media.
        Since media can take many forms such as Mermaid diagrams, tables, images, or URLs, you will completely ignore the
        content of the media and only check whether the media is present in the correct place in the section, has
        the appropriate citation, and proper numbering.
    ...
    """

"""
    ...
    ## CHAIN OF THOUGHT
    
    **Understanding Input:**
    1.1. Read, understand, and compare each section of the expected output and generated output.
    1.2. Since we want to compute scores for each section of the expected output Markdown file, split the expected output
    into sections using the H2 headers as separators.
    
    **Splitting into Sections:**
    2.1. Using the expected output as the reference point, compare each section of the expected and generated
    outputs individually and assign a binary score of 0 or 1, where 0 indicates a mismatch and 1 indicates a perfect match.
    2.2. Always use the expected output as the reference point to extract the sections of interest.
    2.3. When computing the score for an individual section, you will iterate through each section of the expected output,
    find its associated section in the generated output, and compute the score in isolation, ignoring all other sections.
    
    **Assigning Scores to Each Section:**
    3.1. Based on all sections of the expected output, assign a binary score of either 0 or 1
    for all evaluation criteria listed in the instructions:
        - **1:** The generated section matches the expected section perfectly on the given criterion.
        - **0:** The generated section does not match the expected section on the given criterion.
    3.2. Justify why you assigned a score of 0 or 1 with a brief explanation that highlights the reasoning behind the score
    based on the given criterion.
    ...
    """

"""
    ...
    ## WHAT TO AVOID
    
    - Do not provide scores using the generated output as the reference point to divide into sections. You must always
    use the expected output as the reference point to divide into sections.
    - Do not let other sections influence the score of a section. The score of each section must be determined in complete
    isolation from any other section.
    - Do not overlap requirements between different criteria. For example, in the content criterion, as are not interested in the flow
    of ideas, if the ideas are in different order, or something is missing or additional, it's still valid. However, that is an important
    aspect of the flow criterion, which will be invalid.
    ...
    """

"""
    ...
    ## FEW-SHOT EXAMPLES
    
    Here are few-shot examples demonstrating how to compute the scores for each section and criterion:
    <few-shot-examples>
    {examples}
    </few-shot-examples>
    
    ## INPUTS
    
    <generated_output>
    {output}
    </generated_output>
    
    <expected_output>
    {expected_output}
    </expected_output>
    ...
    """

"""
    ...
    Think through your answer step by step, and provide the requested evaluation.
    """

EXAMPLES_DIR = Path(__file__).parent / "examples"

DEFAULT_FEW_SHOT_EXAMPLES = FollowsGTMetricFewShotExamples(
    examples=[
        FollowsGTMetricFewShotExample.from_markdown(
            output_file=EXAMPLES_DIR / "04_structured_outputs" / "article_generated.md",
            expected_output_file=EXAMPLES_DIR / "04_structured_outputs" / "article_ground_truth.md",
            scores=FollowsGTArticleScores(
                sections=[
                    FollowsGTSectionScores(
                        title="Introduction",
                        scores=FollowsGTCriterionScores(
                            content=CriterionScore(score=1, reason="..."),
                            flow=CriterionScore(score=0, reason="..."),
                            structure=CriterionScore(score=1, reason="...")
                    ),
                    ... # The rest of the sections
                    FollowsGTSectionScores(
                        title="References",
                        scores=FollowsGTCriterionScores(
                            content=CriterionScore(score=1, reason="..."),
                            flow=CriterionScore(score=1, reason="..."),
                            structure=CriterionScore(score=1, reason="...")
                    ),
                ]
            ),
        ),
        FollowsGTMetricFewShotExample.from_markdown(
            output_file=EXAMPLES_DIR / "07_reasoning_planning" / "article_generated.md",
            expected_output_file=EXAMPLES_DIR / "07_reasoning_planning" / "article_ground_truth.md",
            scores=FollowsGTArticleScores(
                sections=[
                    FollowsGTSectionScores(
                        title="Introduction",
                        scores=FollowsGTCriterionScores(
                            content=CriterionScore(score=1, reason="..."),
                            flow=CriterionScore(score=0, reason="..."),
                            structure=CriterionScore(score=0, reason="...")
                    ),
                    ... # The rest of the sections
                ]
            ),
       ),
    ]
)

def get_eval_prompt(
    output: str,
    expected_output: str,
    few_shot_examples: FollowsGTMetricFewShotExamples,
) -> str:
    """Generate the evaluation prompt for the follows_gt metric."""
 
    return SYSTEM_PROMPT.format(
        examples=few_shot_examples.to_context(),
        output=output,
        expected_output=expected_output,
    )

from brown.evals.metrics import FollowsGTMetricLLMJudge
from brown.models import SupportedModels
 
EXPECTED_ARTICLE = read_url("https://github.com/iusztinpaul/agentic-ai-engineering-course-data/blob/main/data/inputs/tests/01_sample_small/article.md")
 
OUTPUT_ARTICLE = read_url("https://github.com/iusztinpaul/agentic-ai-engineering-course-data/blob/main/data/inputs/tests/01_sample_small/article_noisy.md")
 
follows_gt_metric = FollowsGTMetricLLMJudge(
    model=SupportedModels.GOOGLE_GEMINI_25_FLASH,
    track=False,  # Disable tracking for this demo
)
 
results = await follows_gt_metric.ascore(
    output=OUTPUT_ARTICLE,
    expected_output=EXPECTED_ARTICLE,
)
 
print(f"\n✓ Generated {len(results)} score results")
for result in results:
    pretty_print.wrapped(f"{result.name} - Score: {result.value:.2f}", title=result.name)
    print("*** REASON ***\n")
    print(f"{result.reason[:500]}..." if len(result.reason) > 500 else result.reason)
    print("\n")

✓ Generated 3 score results

------------------------------ follows_gt_content -------------------------------
  follows_gt_content - Score: 0.60
---------------------------------------------------------------------------------
*** REASON ***

Introduction:
**1:** Both sections cover the same core subjects and ideas, discussing the critical architectural decision between LLM workflows and AI agents and the lesson's objectives.

Understanding the Spectrum: From Workflows to Agents:
**0:** Both sections introduce LLM workflows and AI agents with their core properties and diagrams. However, the generated section's description of 'LLM Workflows' is significantly less detailed than the expected output, missing key characteristics like dete...

--------------------------------- follows_gt_flow -------------------------------
  follows_gt_flow - Score: 0.60
---------------------------------------------------------------------------------
*** REASON ***

Introduction:
**0:** The generated section reverses the order of the introductory paragraph and the paragraph stating the lesson's objectives compared to the expected output.

Understanding the Spectrum: From Workflows to Agents:
**1:** Both sections follow the same logical flow, introducing the spectrum, then defining LLM workflows and AI agents with their respective diagrams.

...

------------------------------- follows_gt_structure ----------------------------
  follows_gt_structure - Score: 0.60
---------------------------------------------------------------------------------
*** REASON ***

Introduction:
**1:** Both sections maintain a similar paragraph structure and do not use an H2 header for the introduction.

Understanding the Spectrum: From Workflows to Agents:
**1:** Both sections maintain the same H2 and H3 heading structure, paragraph length patterns, and consistent formatting for Mermaid diagrams and their citations.

...

class UserIntentMetricLLMJudge(BrownBaseMetric):
        """A metric that evaluates how well generated articles follow article guidelines and are anchored in research."""
    
        def __init__(
            self,
            model: SupportedModels = SupportedModels.GOOGLE_GEMINI_25_FLASH,
            name: str = "user_intent",
            model_config: ModelConfig | None = None,
            track: bool = True,
            project_name: str | None = None,
    ) -> None:
            model_config = model_config or ModelConfig(
                temperature=0.0, thinking_budget=1024 * 4, include_thoughts=False, max_retries=3
            )
            super().__init__(
                model=model,
                name=name,
                structured_output_type=UserIntentArticleScores,
                few_shot_examples=prompts.DEFAULT_FEW_SHOT_EXAMPLES,
                model_config=model_config,
                track=track,
                project_name=project_name,
            )

async def ascore(
            self,
            input: str,
            context: dict[str, Any],
            output: str,
            **ignored_kwargs: Any,
    ) -> score_result.ScoreResult | list[score_result.ScoreResult]:
            # Extract research from context
            if "research" not in context:
                raise ValueError("Context must contain a 'research' key with research content")
    
            research_content = context["research"]
            model_client = self.init_model()
    
            llm_query = prompts.get_eval_prompt(
                input=input,
                context=research_content,
                output=output,
                few_shot_examples=self.few_shot_examples,
            )
    
            user_intent_response = cast(
                UserIntentArticleScores,
                await model_client.ainvoke([{"role": "user", "content": llm_query}]),
            )
    
            if not user_intent_response:
                raise ValueError("Model failed to return a structured response.")
    
            return user_intent_response.to_score_result(self.name)

class UserIntentCriteriaScores(CriteriaScores):
        """Represents scores for both evaluation dimensions of a section."""
    
        guideline_adherence: CriterionScore
        research_anchoring: CriterionScore
    
    class UserIntentSectionScores(pydantic.BaseModel):
        """Section evaluation with scores across all UserIntent dimensions."""
    
        title: str = pydantic.Field(description="The title of the section being evaluated.")
        scores: UserIntentCriteriaScores = pydantic.Field(description="The scores of the section.")
    
    class UserIntentArticleScores(pydantic.BaseModel):
        """Article-level scores for the UserIntent evaluation metric."""
    
        sections: list[UserIntentSectionScores]
    
        def to_score_result(self, prefix: str) -> list[score_result.ScoreResult]:
            return aggregate_section_scores_to_results(self.sections, prefix)

class UserIntentMetricFewShotExample(BaseExample):
        """Represents a single example for the user intent evaluation."""
    
        input: str  # Article guideline
        context: str  # Research
        output: str  # Generated article
        scores: UserIntentArticleScores
    
        @classmethod
        def from_markdown(
            cls,
            input_file: Path,
            context_file: Path,
            output_file: Path,
            scores: UserIntentArticleScores,
    ) -> "UserIntentMetricFewShotExample":
            input_content = input_file.read_text()
            context_content = context_file.read_text()
            output_content = output_file.read_text()
            return cls(input=input_content, context=context_content, output=output_content, scores=scores)
    
        def to_context(self) -> str:
            return f"""
    <input>
    {self.input}
    </input>
    <context>
    {self.context}
    </context>
    <output>
    {self.output}
    </output>
    {self.scores.to_context()}
    """

class UserIntentMetricFewShotExamples(BaseFewShotExamples[UserIntentMetricFewShotExample]):
        """Collection of few-shot examples for the UserIntent evaluation metric."""
        pass

SYSTEM_PROMPT = """
    You are an expert in Natural Language Processing (NLP) evaluation metrics, specifically trained to
    assess how well generated content follows specific guidelines and incorporates provided research material.
    
    The AI system you will evaluate takes as input an article guideline (<input>) and research material (<context>), and
    based on these inputs, another large language model (LLM) generates an article (<output>).
    
    Your task is to evaluate whether the generated article (<output>) adheres to the given
    user intent structured as an article guideline and is properly anchored in the provided research.
    ...
    """

"""
    ...
    ## INSTRUCTIONS
    
    1. You must analyze the given article guideline (<input>), research material (<context>), and
    generated article (<output>) to determine how well the output follows the input guidelines and utilizes the research.
    2. The input, context, and output are in Markdown format.
    3. Instead of comparing the outputs as a whole, you will divide the outputs into sections and compare each section
    individually.
    4. Since the article guideline reflects the user expectations and intent, you will always use it as the reference point
    to understand which sections from the generated article should be evaluated. To find the expected sections, look within
    the given article guideline for the keyword "Outline" or actual section titles often marked with H2 headers prefixed
    with "Section". You will always use these as the anchor points, the expected sections, when making comparisons between
    sections.
    5. When associating a section from the article guideline with one from the generated article, you will first look
    for a matching or similar title. If there is no match based solely on the title, you will try to make associations based
    on the content of the section. For example, if the expected section has 3 points on how RAG works and
    the generated section has 3 points on how RAG works as well, then they are associated even if the titles are different.
    If a required section mentioned in the guideline is missing from the generated article, you will assign a score of
    0 to all evaluation criteria.
    6. Using the article guideline as an anchor, you will divide the generated article into sections and evaluate each section
    individually against all given criteria.
    7. Sections are divided by H2 headers, marked as "##" in Markdown. You will use the headers as separators.
    Anything between two H2 headers constitutes a section. The only valid exception to this rule is the first section,
    the introduction, which sometimes appears between the title and the first H2 header. You will never include the title or
    subtitle as part of the first section.
    ...
    """

"""
    ...
    8. The score can only be an integer value of 0 or 1. For each section, you will assign a binary integer score (0 or 1) based on
    two criteria:
       1. **Guideline Adherence**: For each expected section in the article guideline, you will evaluate whether the generated
       section follows the specific section requirements outlined in the article guideline:
            - We expect a perfect match between the expected section and the generated section. Intuitively, you can
            think of the section guideline as a sketch, a compressed version of the generated section.
            - Less: If any topic from the expected article guideline is missing from the generated article, you will assign
            a score of 0.
            - More: If the generated section has any additional topics that are not in the expected article guideline,
            you will assign a score of 0.
            - Different Order: If the order of ideas from the expected article guideline is not followed in the
            generated article, you will assign a score of 0.
            - If section constraints are provided, we are looking only for a rough approximation of the length. The exact
            section length criterias are present in the article guideline. Errors of ±100 units are acceptable. Units can
            be words, characters, or reading time. For example, if the expected section length is 100 words and the generated section length
            is 190 words, you will assign a score of 1. But if the generated section is 230 words, as it exceeds the tolerance range,
            you will assign a score of 0.
    ...
    """

"""
    ...
       2. **Research Anchoring**: For each expected section in the article guideline, you will evaluate whether the generated
       section content is based on or derived from the provided research:
            - We expect each section from the generated article to be generated entirely based on the ideas provided
            in the article guideline and research. Thus, you can consider both the context and article guideline as the
            "research", the single source of truth.
            - If any idea from the generated section is not present in the research, you will assign a score of 0.
            - The generated section does not have to contain all the ideas from the research, just a subset of them.
            - If no research is explicitly referenced through citations, you will manually check if the generated section content
            it's based solely on the research. Missing explicit citations is valid. What it's critical is all the ideas to
            adhere to the research. Thus, if the generated section content is based solely on the research, while missing citations,
            you will assign a score of 1.
    ...
    """

"""
    ...
    9. Along with the binary score, you will provide a brief and concise explanation containing the reasoning behind
    the score for each criterion. The score will be used to debug and monitor the evaluation process. Therefore, it is
    important to provide thorough reasoning for the score. Since we provide binary scores, the reasoning should always
    contain what is good and what is problematic about the generated section, regardless of the score. For example, if the score
    is 0, the reasoning should also contain what is good about the generated section, such as "the generated section
    contains all the bulleted points from the expected section guideline," and what is problematic, such as "however, it contains an
    additional section on AI Evals that is not present in the guideline." Also, when generating the reasoning for the
    research anchoring criterion, you will always mention if the topic comes from the article guideline, context, or both, while
    supporting every single claim with evidence from the research. For example, the generated section is correctly anchored in the research,
    while the fundamentals on RAG are based on the context, while the specific details on the RAG architecture are based on the article
    guideline.
    10. Important rules when evaluating:
       - Focus on substance, not superficial formatting differences
       - When comparing **media**, you only care about the placement and the caption of the media.
        Since media can take many forms such as Mermaid diagrams, images, or URLs, you will completely ignore the
        content of the media. Based on the section guideline, you will check whether the media is present in the
        correct place. Based on the caption of the media, you will check whether it is properly anchored in the research.
    ...
    """

"""
    ...
    ## CHAIN OF THOUGHT
    
    **Understanding Input:**
    1.1. Read and understand the article guideline (<input>) to identify specific requirements, structure,
    content expectations, constraints, and most importantly the expected sections.
    1.2. Read and understand the context (<context>) to identify available information, sources, and key findings.
    1.3. Label the article guideline and context as the "research", the single source of truth.
    1.4. Read and understand the generated article (<output>) and split it into sections using H2 headers as separators.
    1.5. Connect the expected sections from the article guideline to the sections from the generated article.
    
    **Section-by-Section Evaluation:**
    2.1. For each section identified in the article guideline, locate its associated section in the generated article, and
    evaluate it against both criteria. If a section is found in the article guideline and is missing in the generated
    article, you will assign a score of 0 to all evaluation criteria.
    2.2. Evaluate guideline adherence between each expected section from the article guideline and the associated section
    from the generated article.
    2.3. Evaluate research anchoring by first selecting the sections to evaluate from the article guideline and then
    comparing the generated section to the research, found in the context and the associated section guideline.
    
    **Assigning Scores:**
    3.1. Based on each section expected from the article guideline, assign a binary score of either 0 or 1
    for all evaluation criteria listed in the instructions:
        - Score 1 if the section clearly follows the requirements detailed in the instructions.
        - Score 0 if it fails to follow the requirements detailed in the instructions.
    3.2. Justify why you assigned a score of 0 or 1 with a brief explanation that highlights the reasoning behind the score
    based on the given criterion.
    ...
    """

"""
    ...
    ## WHAT TO AVOID
    
    - Do not provide scores using the generated output as the reference point to divide into sections. You must always
    use the article guideline as the reference point to divide into sections.
    - Do not let other sections influence the score of a section. The score of each section must be determined in complete
    isolation from any other section.
    ...
    """

"""
    ...
    ## FEW-SHOT EXAMPLES
    
    Here are few-shot examples demonstrating how to compute the scores for each section and criterion:
    <few-shot-examples>
    {examples}
    </few-shot-examples>
    
    ## INPUTS
    
    <input>
    {input}
    </input>
    
    <context>
    {context}
    </context>
    
    <output>
    {output}
    </output>
    ...
    """

"""
    ...
    Think through your answer step by step, and provide the requested evaluation.
    """

EXAMPLES_DIR = Path(__file__).parent / "examples"

DEFAULT_FEW_SHOT_EXAMPLES = UserIntentMetricFewShotExamples(
    examples=[
        UserIntentMetricFewShotExample.from_markdown(
            input_file=EXAMPLES_DIR / "04_structured_outputs" / "article_guideline.md",
            context_file=EXAMPLES_DIR / "04_structured_outputs" / "research.md",
            output_file=EXAMPLES_DIR / "04_structured_outputs" / "article_generated.md",
            scores=UserIntentArticleScores(
                sections=[
                  UserIntentSectionScores(
                    title="Introduction",
                      scores=UserIntentCriteriaScores(
                        guideline_adherence=CriterionScore(score=0, reason="...")
                        research_anchoring=CriterionScore(score=0, reason="...")
                        ),
                    ),
                    ... # The rest of the sections
                ]
            ),
        ),
        UserIntentMetricFewShotExample.from_markdown(
            input_file=EXAMPLES_DIR / "07_reasoning_planning" / "article_guideline.md",
            context_file=EXAMPLES_DIR / "07_reasoning_planning" / "research.md",
            output_file=EXAMPLES_DIR / "07_reasoning_planning" / "article_generated.md",
            scores=UserIntentArticleScores(
                sections=[
                  UserIntentSectionScores(
                    title="Introduction",
                      scores=UserIntentCriteriaScores(
                        guideline_adherence=CriterionScore(score=1, reason="...")
                        research_anchoring=CriterionScore(score=1, reason="...")
                        ),
                    ),
                    ... # The rest of the sections
                ]
            ),
        ),
    ]
)

def get_eval_prompt(
    input: str,
    context: str,
    output: str,
    few_shot_examples: UserIntentMetricFewShotExamples,
) -> str:
    """Generate the evaluation prompt for the user intent metric."""
 
    return SYSTEM_PROMPT.format(
        examples=few_shot_examples.to_context(),
        input=input,
        context=context,
        output=output,
    )

from brown.evals.metrics import UserIntentMetricLLMJudge
from brown.models import SupportedModels
 
ARTICLE_GUIDELINE = read_url("https://github.com/iusztinpaul/agentic-ai-engineering-course-data/blob/main/data/inputs/tests/01_sample_small/article_guideline.md")
 
RESEARCH = read_url("https://github.com/iusztinpaul/agentic-ai-engineering-course-data/blob/main/data/inputs/tests/01_sample_small/research.md")
 
OUTPUT_ARTICLE = read_url("https://github.com/iusztinpaul/agentic-ai-engineering-course-data/blob/main/data/inputs/tests/01_sample_small/article_noisy.md")
 
user_intent_metric = UserIntentMetricLLMJudge(
    model=SupportedModels.GOOGLE_GEMINI_25_FLASH,
    track=False,  # Disable tracking for this demo
)
 
results = await user_intent_metric.ascore(
    input=ARTICLE_GUIDELINE,
    context={"research": RESEARCH},
    output=OUTPUT_ARTICLE,
)
 
print(f"\n✓ Generated {len(results)} score results")
for result in results:
    pretty_print.wrapped(f"{result.name} - Score: {result.value:.2f}", title=result.name)
    print("*** REASON ***\n")
    print(f"{result.reason[:1000]}..." if len(result.reason) > 1000 else result.reason)
    print("\n")

✓ Generated 2 score results

------------------------- user_intent_guideline_adherence -----------------------
  user_intent_guideline_adherence - Score: 0.50
---------------------------------------------------------------------------------
*** REASON ***

Introduction: The Critical Decision Every AI Engineer Faces:
**1:** The section correctly introduces the problem of architectural decisions in AI, and provides a quick walkthrough of what will be learned. The length is approximately 90 words, which is within the 100-word limit (allowing for a 10% tolerance).

Understanding the Spectrum: From Workflows to Agents:
**1:** The section correctly defines LLM workflows and AI agents, focusing on their properties and usage rather than technical specifics. It includes both required Mermaid diagrams and adheres to the approximate 200-word length constraint (excluding diagrams).

Choosing Your Path:
**0:** The section fails to adhere to the guideline's instruction to explore the core difference as 'Developer-defined logic vs LLM-driven autonomy in reasoning and action selection.' Instead, it incorrectly states the core difference is 'use RAG or not.' While it includes the required image and discusses hybrid approaches, this fundamental misreprese...


--------------------------- user_intent_research_anchoring ----------------------
  user_intent_research_anchoring - Score: 0.75
---------------------------------------------------------------------------------
*** REASON ***

Introduction: The Critical Decision Every AI Engineer Faces:
**1:** The content of this introductory section is high-level and aligns with the overall framing of the lesson as presented in the guideline. It does not introduce any specific factual claims that require external research anchoring, and therefore contains no information that contradicts the provided research material.

Understanding the Spectrum: From Workflows to Agents:
**1:** The definitions and characteristics of LLM workflows and AI agents are well-anchored in the provided research. The concept of workflows as 'predefined code paths' and agents as 'dynamically decides' is consistently supported by sources [1], [2], and [3] in the output's references, which are derived from the research material.

Choosing Your Path:
**0:** The section introduces a factual inaccuracy by stating the core difference between workflows and agents is 'use RAG or not.' This contradicts the research, which consistently defines the core differe...

def build_evaluation_metrics(
        metrics: list[str], model: SupportedModels, model_config: ModelConfig | None = None
    ) -> list[base_metric.BaseMetric]:
        """Get evaluation metrics based on the provided metric names."""
        metrics_mapping = {
            "user_intent": UserIntentMetricLLMJudge(model=model, model_config=model_config),
            "follows_gt": FollowsGTMetricLLMJudge(model=model, model_config=model_config),
        }
    
        try:
            return [metrics_mapping[metric] for metric in metrics]
        except KeyError as e:
            raise ValueError(f"Unknown metric name: {e}")

from brown.evals.metrics import build_evaluation_metrics
 
all_metrics = build_evaluation_metrics(
    metrics=["user_intent", "follows_gt"],
    model=SupportedModels.GOOGLE_GEMINI_25_FLASH,
)
 
print("Created metrics:")
for metric in all_metrics:
    print(f"  - {metric.name}: {metric.__class__.__name__}")

Created metrics:
  - user_intent: UserIntentMetricLLMJudge
  - follows_gt: FollowsGTMetricLLMJudge

from langchain_core.runnables.config import RunnableConfig
    
    from brown.builders import build_short_term_memory
    from brown.config_app import get_app_config
    from brown.evals.dataset import EvalSampleDict
    from brown.observability import tracing
    from brown.utils import a
    from brown.workflows import build_generate_article_workflow
    
    @a.as_sync
    async def evaluation_task(
        sample: EvalSampleDict,
        cache_dir: Path,
        read_from_cache: bool = False,
        clean_cache: bool = False,
        debug: bool = False,
    ) -> Dict[str, Any]:
        """Generate an article using the Brown agent for a single evaluation sample."""
        logger.info(f"Processing evaluation sample: {sample['name']}")
    
        cache_dir = cache_dir / sample["directory"]
        cache_dir.mkdir(parents=True, exist_ok=True)
    
        try:
            article_guideline_path = cache_dir / "article_guideline.md"
            research_path = cache_dir / "research.md"
    
            article_guideline_path.write_text(sample["article_guideline"], encoding="utf-8")
            research_path.write_text(sample["research"], encoding="utf-8")

thread_id = str(uuid.uuid4())
            inputs = {"dir_path": cache_dir}
            tracer = tracing.build_handler(thread_id, tags=["generate-evaluation"])
            config = RunnableConfig(
                configurable={"thread_id": thread_id},
                callbacks=[tracer],
            )
    
            try:
                generated_article = await __run(config, inputs, read_from_cache)
            except Exception:
                generated_article = "ERROR: Failed to generate article"
                logger.exception(f"Failed to generate article for sample `{sample['name']}`")
    
            return {
                "input": sample["article_guideline"],
                "context": {"research": sample["research"], "debug": debug},
                "output": generated_article,
                "expected_output": sample["ground_truth_article"],
                "reference": sample["ground_truth_article"],
                "name": sample["name"],
            }
        finally:
            if clean_cache and cache_dir.exists():
                shutil.rmtree(cache_dir)

async def __run(config: RunnableConfig, inputs: Dict[str, Any], read_from_cache: bool = False) -> str:
        """Run the article generation workflow and extract the final article."""
    
        app_config = get_app_config()
    
        article_path = inputs["dir_path"] / "article.md"
        if read_from_cache:
            assert article_path.exists(), f"Article file not found in cache at `{article_path}`"
            logger.success(f"Successfully read article from cache at `{article_path}`")
        else:
            async with build_short_term_memory(app_config) as checkpointer:
                generate_article_workflow = build_generate_article_workflow(checkpointer=checkpointer)
                await generate_article_workflow.ainvoke(inputs, config)
    
            logger.success(f"Successfully generated article at `{article_path}`")
    
        article = article_path.read_text(encoding="utf-8")
    
        return article

from typing import Any, Callable, Literal
    
    from loguru import logger
    from opik import evaluation
    from opik.evaluation.metrics import base_metric
    
    from brown.config import get_settings
    from brown.config_app import get_app_config as load_app_config
    
    from .opik_utils import get_dataset
    
    app_config = load_app_config()
    
    def evaluate(
        dataset_name: str,
        metrics: list[base_metric.BaseMetric],
        evaluation_task: Callable,
        llm_judge_config: dict[str, Any],
        workers: int = 2,
        nb_samples: int | None = None,
        dataset_item_names: list[str] | None = None,
    ) -> None:
        """Run an Opik evaluation with the provided metrics and task."""
    
        assert get_settings().OPIK_API_KEY, "OPIK_API_KEY is not set."
    
        dataset = get_dataset(dataset_name)
        if not dataset:
            raise ValueError(f"Dataset '{dataset_name}' does not exist.")
    
        if dataset_item_names:
            all_dataset_items = dataset.get_items()
            dataset_item_ids = [item["id"] for item in all_dataset_items if item["name"] in dataset_item_names]
            logger.info(f"Evaluating {len(dataset_item_ids)}/{len(all_dataset_items)} dataset items.")
        else:
            dataset_item_ids = None
    
        logger.info("Starting evaluation...")
    
        llm_judge_config = {
            "dataset_name": dataset.name,
            "llm_judge_config": llm_judge_config,
            "app_config": app_config.model_dump(mode="json"),
        }
    
        evaluation.evaluate(
            dataset=dataset,
            task=evaluation_task,
            scoring_metrics=metrics,
            experiment_config=llm_judge_config,
            task_threads=workers,
            nb_samples=nb_samples,
            dataset_item_ids=dataset_item_ids,
        )

def get_dataset(name: str) -> opik.Dataset | None:
        """Get a dataset by name."""
        client = opik.Opik()
        try:
            dataset = client.get_dataset(name=name)
        except Exception:
            dataset = None
        return dataset

def create_evaluation_task(
        cache_dir: Path,
        read_from_cache: bool = False,
        clean_cache: bool = False,
        debug: bool = False,
    ) -> Callable:
        """Create a reusable evaluation task with fixed runtime parameters."""
        return partial(
            evaluation_task,
            cache_dir=cache_dir,
            read_from_cache=read_from_cache,
            clean_cache=clean_cache,
            debug=debug,
        )

DATASET_NAME = "brown-course-lessons"
CACHE_DIR = OUTPUTS_DIR / "evals-pro" # Here, we store the generated articles from the entire dataset to avoid recomputing them for this example.
METRICS_TO_USE = ["follows_gt", "user_intent"]
WORKERS = 1

evaluation_task = create_evaluation_task(
    cache_dir=CACHE_DIR,
    read_from_cache=True,
    clean_cache=False,
    debug=False,
)

model = SupportedModels.GOOGLE_GEMINI_25_FLASH
model_config = ModelConfig(temperature=0.0, thinking_budget=int(1024 * 0.5), include_thoughts=False, max_retries=3)
evaluation_metrics = build_evaluation_metrics(METRICS_TO_USE, model, model_config)

val_dataset_item_names = ["Lesson 10: Memory"]
test_dataset_item_names = [
    "Lesson 2: Workflows vs. Agents",
    "Lesson 3: Context Engineering",
    "Lesson 5: Workflow Patterns",
    "Lesson 6: Tools",
    "Lesson 8: ReAct Practice",
    "Lesson 9: Retrieval-Augmented Generation (RAG)",
    "Lesson 11: Multimodal Data",
]

SPLITS = {
    "val": val_dataset_item_names,
    "test": test_dataset_item_names,
}

split = "val"

logger.info(f"Starting evaluation with dataset: {DATASET_NAME} on the `{split}` split")
evaluate(
    dataset_name=DATASET_NAME,
    metrics=evaluation_metrics,
    evaluation_task=evaluation_task,
    llm_judge_config={"model": str(model), **model_config.model_dump()},
    workers=WORKERS,
    dataset_item_names=SPLITS[split],
    split=split,
)

split = "test"

logger.info(f"Starting evaluation with dataset: {DATASET_NAME} on the `{split}` split")
evaluate(
    dataset_name=DATASET_NAME,
    metrics=evaluation_metrics,
    evaluation_task=evaluation_task,
    llm_judge_config={"model": str(model), **model_config.model_dump()},
    workers=WORKERS,
    dataset_item_names=SPLITS[split],
    split=split,
)

╭─ brown-course-lessons (7 samples) ────────────╮
│                                               │
│ Total time:        00:03:03                   │
│ Number of samples: 7                          │
│                                               │
│ follows_gt_content: 0.8962 (avg)              │
│ follows_gt_flow: 0.4127 (avg)                 │
│ follows_gt_structure: 0.1086 (avg)            │
│ user_intent_guideline_adherence: 0.7468 (avg) │
│ user_intent_research_anchoring: 0.8075 (avg)  │
│                                               │
╰───────────────────────────────────────────────╯