Skip to content

DeepEval

DeepEvalBase

Bases: MetricBase

Base class for DeepEval metrics.

Here we use AzureChatOpenAI interface; replace if needed.

Source code in ragfoundry/evaluation/deep.py 
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
class DeepEvalBase(MetricBase):
    """
    Base class for DeepEval metrics.

    Here we use AzureChatOpenAI interface; replace if needed.
    """

    def __init__(
        self,
        key_names: dict,
        api_version,
        azure_endpoint,
        azure_deployment,
        **kwargs,
    ):
        super().__init__(key_names, **kwargs)
        from deepeval.test_case import LLMTestCase
        from langchain_openai import AzureChatOpenAI

        self.local = True
        self.query = self.key_names["query"]
        self.context = self.key_names["context"]
        self.test_case = LLMTestCase

        self.model = AzureChatOpenAI(
            api_version=api_version,
            azure_endpoint=azure_endpoint,
            azure_deployment=azure_deployment,
            request_timeout=600,
            max_retries=10,
        )

Faithfulness

Bases: DeepEvalBase

Faithfulness metric from DeepEval, based on RAGAS.

Measures faithfulness of generated text by comparing it to the target text.

Source code in ragfoundry/evaluation/deep.py 
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
class Faithfulness(DeepEvalBase):
    """
    Faithfulness metric from DeepEval, based on RAGAS.

    Measures faithfulness of generated text by comparing it to the target text.
    """

    def __init__(self, key_names: dict, threshold=0.3, **kwargs):
        super().__init__(key_names, **kwargs)
        from deepeval.metrics.ragas import RAGASFaithfulnessMetric

        self.metric = RAGASFaithfulnessMetric(threshold=threshold, model=self.model)

    def measure(self, example):
        query = example[self.query]
        output = example[self.field]
        context = example[self.context]

        test_case = self.test_case(
            input=query,
            actual_output=output or "No answer.",
            retrieval_context=[context] if isinstance(context, str) else context,
        )
        try:
            self.metric.measure(test_case)
            score = self.metric.score
        except Exception as e:
            logging.error(f"OpenAI exception: {e}")
            score = 0

        return {"faithfulness": score if not math.isnan(score) else 0}

Hallucination

Bases: DeepEvalBase

Hallucination metric from DeepEval.

Measures hallucination of generated text by comparing it to the retrieved documents.

Source code in ragfoundry/evaluation/deep.py 
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
class Hallucination(DeepEvalBase):
    """
    Hallucination metric from DeepEval.

    Measures hallucination of generated text by comparing it to the retrieved documents.
    """

    def __init__(self, key_names: dict, threshold=0.5, **kwargs):
        super().__init__(key_names, **kwargs)
        from deepeval.metrics import HallucinationMetric

        self.metric = HallucinationMetric(
            threshold=threshold, include_reason=False, model=self.model
        )

    def measure(self, example):
        output = example[self.field]
        context = example[self.context]

        test_case = self.test_case(
            input="",
            actual_output=output,
            context=[context] if isinstance(context, str) else context,
        )

        try:
            self.metric.measure(test_case)
            score = self.metric.score
        except Exception as e:
            logging.error(f"OpenAI exception: {e}")
            score = 0

        return {"hallucination": score}

Relevancy

Bases: DeepEvalBase

Answer relevancy metric from DeepEval, based on RAGAS.

Measures relevancy of generated text by comparing it to the retrieved documents.

Source code in ragfoundry/evaluation/deep.py 
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
class Relevancy(DeepEvalBase):
    """
    Answer relevancy metric from DeepEval, based on RAGAS.

    Measures relevancy of generated text by comparing it to the retrieved documents.
    """

    def __init__(self, key_names: dict, embeddings, threshold=0.3, **kwargs):
        super().__init__(key_names, **kwargs)
        from deepeval.metrics.ragas import RAGASAnswerRelevancyMetric
        from ragas.embeddings import HuggingfaceEmbeddings

        self.metric = RAGASAnswerRelevancyMetric(
            threshold=threshold,
            embeddings=HuggingfaceEmbeddings(model_name=embeddings),
            model=self.model,
        )

    def measure(self, example):
        query = example[self.query]
        output = example[self.field]
        context = example[self.context]

        test_case = self.test_case(
            input=query,
            actual_output=output or "No answer.",
            retrieval_context=[context] if isinstance(context, str) else context,
        )
        try:
            self.metric.measure(test_case)
            score = self.metric.score
        except Exception as e:
            logging.error(f"OpenAI exception: {e}")
            score = 0

        return {"relevancy": score}