Skip to content

judge

Judge

Class to create judge inputs for a list of completed responses.

Parameters:

Name Type Description Default
completed_responses list[dict]

A list of dictionaries containing the responses to judge. Each dictionary should contain the keys “prompt”, and “response”

 required
template_prompts dict[str, str]

A dictionary containing the template prompt strings to be used for the judge LLMs. The keys should be the name of the template and the value should be the template. The string templates (the values) are to be used to format the prompt for the judge LLMs. Often contains placeholders for the input prompt (INPUT_PROMPT) and the output response (OUTPUT_RESPONSE) which will be formatted with the prompt and response from the completed prompt dict

 required
judge_settings dict

A dictionary of judge settings with the keys “api”, “model_name”, “parameters”. Used to define the judge LLMs to be used in the judging process

 required
Source code in src/prompto/judge.py 
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
class Judge:
    """
    Class to create judge inputs for a list of completed responses.

    Parameters
    ----------
    completed_responses : list[dict]
        A list of dictionaries containing the responses to judge.
        Each dictionary should contain the keys "prompt",
        and "response"
    template_prompts : dict[str, str]
        A dictionary containing the template prompt strings
        to be used for the judge LLMs. The keys should be the
        name of the template and the value should be the template.
        The string templates (the values) are to be used to format
        the prompt for the judge LLMs. Often contains placeholders
        for the input prompt (INPUT_PROMPT) and the
        output response (OUTPUT_RESPONSE) which will be formatted
        with the prompt and response from the completed prompt dict
    judge_settings : dict
        A dictionary of judge settings with the keys "api",
        "model_name", "parameters". Used to define the
        judge LLMs to be used in the judging process
    """

    def __init__(
        self,
        completed_responses: list[dict],
        template_prompts: dict[str, str],
        judge_settings: dict,
    ):
        if not isinstance(template_prompts, dict):
            raise TypeError("template_prompts must be a dictionary")
        self.check_judge_settings(judge_settings)
        self.completed_responses = completed_responses
        self.template_prompts = template_prompts
        self.judge_settings = judge_settings
        self.judge_prompts: list[dict] = []

    @staticmethod
    def check_judge_settings(judge_settings: dict[str, dict]) -> bool:
        """
        Method to check if the judge settings dictionary is valid.

        Parameters
        ----------
        judge_settings : dict
            A dictionary of judge settings with the keys "api",
            "model_name", "parameters". Used to define the
            judge LLMs to be used in the judging process

        Returns
        -------
        bool
            True if the judge settings dictionary is valid.
            Errors will be raised if the dictionary is invalid
        """
        if not isinstance(judge_settings, dict):
            raise TypeError("judge_settings must be a dictionary")

        for judge, settings in judge_settings.items():
            if not isinstance(settings, dict):
                raise TypeError(f"Value for judge key '{judge}' must be a dictionary")
            if "api" not in settings:
                raise KeyError(f"'api' key not found in settings for judge '{judge}'")
            if "model_name" not in settings:
                raise KeyError(
                    f"'model_name' key not found in settings for judge '{judge}'"
                )
            if "parameters" not in settings:
                raise KeyError(
                    f"'parameters' key not found in settings for judge '{judge}'"
                )
            if not isinstance(settings["parameters"], dict):
                raise TypeError(
                    f"Value for 'parameters' key must be a dictionary for judge '{judge}'"
                )

        return True

    @staticmethod
    def check_judge_in_judge_settings(
        judge: str | list[str], judge_settings: dict[str, dict]
    ) -> bool:
        """
        Method to check if the judge(s) are in the judge settings dictionary.

        Parameters
        ----------
        judge : str | list[str]
            A single judge or a list of judges to check if they
            are keys in the judge_settings dictionary
        judge_settings : dict[str, dict]
            A dictionary of judge settings with the keys "api",
            "model_name", "parameters". Used to define the
            judge LLMs to be used in the judging process

        Returns
        -------
        bool
            True if the judge(s) are in the judge settings dictionary.
            Errors will be raised if the judge(s) are not in the dictionary
        """
        if isinstance(judge, str):
            judge = [judge]

        for j in judge:
            if not isinstance(j, str):
                raise TypeError("If judge is a list, each element must be a string")
            if j not in judge_settings.keys():
                raise KeyError(f"Judge '{j}' is not a key in judge_settings")

        return True

    def create_judge_inputs(self, judge: list[str] | str) -> list[dict]:
        """
        Method to create a list of input prompt dictionaries to
        be processed by the judge LLM(s).

        Parameters
        ----------
        judge : list[str] | str
            A list of judges or a single judge to be used to.
            These must be keys in the judge settings dictionary,
            otherwise a KeyError will be raised

        Returns
        -------
        list[dict]
            A list of dictionaries containing the input prompt
            for the judge LLM(s). Each dictionary will contain a
            new prompt for each prompt/response pair in the
            completed_responses list using the template_prompt
        """
        if isinstance(judge, str):
            judge = [judge]

        assert self.check_judge_in_judge_settings(
            judge=judge, judge_settings=self.judge_settings
        )

        self.judge_prompts = []
        for j in judge:
            for template_name, template_prompt in self.template_prompts.items():
                self.judge_prompts += [
                    {
                        "id": f"judge-{j}-{template_name}-{str(response.get('id', 'NA'))}",
                        "template_name": template_name,
                        "prompt": template_prompt.format(
                            INPUT_PROMPT=response["prompt"],
                            OUTPUT_RESPONSE=response["response"],
                        ),
                    }
                    | self.judge_settings[j]
                    | {f"input-{k}": v for k, v in response.items()}
                    for response in tqdm(
                        self.completed_responses,
                        desc=f"Creating judge inputs for judge '{j}' and template '{template_name}'",
                        unit="responses",
                    )
                ]

        return self.judge_prompts

    def create_judge_file(
        self, judge: list[str] | str, out_filepath: str
    ) -> list[dict]:
        """
        Method to create a judge file (as a jsonl file) containing
        the input prompt dictionaries to be processed by the judge LLM(s).

        Parameters
        ----------
        judge : list[str] | str
            A list of judges or a single judge to be used to.
            These must be keys in the judge settings dictionary,
            otherwise a KeyError will be raised
        out_filepath : str
            The path to the output file where the judge inputs
            will be saved as a jsonl file

        Returns
        -------
        list[dict]
            A list of dictionaries containing the input prompt
            for the judge LLM(s). Each dictionary will contain a
            new prompt for each prompt/response pair in the
            completed_responses list using the template_prompt
        """
        if not out_filepath.endswith(".jsonl"):
            raise ValueError("out_filepath must end with '.jsonl'")

        judge_prompts = self.create_judge_inputs(judge=judge)

        logging.info(f"Creating judge file at {out_filepath}...")
        with open(out_filepath, "w", encoding="utf-8") as f:
            for j_input in tqdm(
                judge_prompts,
                desc=f"Writing judge prompts to {out_filepath}",
                unit="prompts",
            ):
                json.dump(j_input, f)
                f.write("\n")

        return judge_prompts

check_judge_in_judge_settings staticmethod 

check_judge_in_judge_settings(
    judge: str | list[str], judge_settings: dict[str, dict]
) -> bool

Method to check if the judge(s) are in the judge settings dictionary.

Parameters:

Name Type Description Default
judge str | list[str]

A single judge or a list of judges to check if they are keys in the judge_settings dictionary

 required
judge_settings dict[str, dict]

A dictionary of judge settings with the keys “api”, “model_name”, “parameters”. Used to define the judge LLMs to be used in the judging process

 required

Returns:

Type Description
bool

True if the judge(s) are in the judge settings dictionary. Errors will be raised if the judge(s) are not in the dictionary
Source code in src/prompto/judge.py 
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
@staticmethod
def check_judge_in_judge_settings(
    judge: str | list[str], judge_settings: dict[str, dict]
) -> bool:
    """
    Method to check if the judge(s) are in the judge settings dictionary.

    Parameters
    ----------
    judge : str | list[str]
        A single judge or a list of judges to check if they
        are keys in the judge_settings dictionary
    judge_settings : dict[str, dict]
        A dictionary of judge settings with the keys "api",
        "model_name", "parameters". Used to define the
        judge LLMs to be used in the judging process

    Returns
    -------
    bool
        True if the judge(s) are in the judge settings dictionary.
        Errors will be raised if the judge(s) are not in the dictionary
    """
    if isinstance(judge, str):
        judge = [judge]

    for j in judge:
        if not isinstance(j, str):
            raise TypeError("If judge is a list, each element must be a string")
        if j not in judge_settings.keys():
            raise KeyError(f"Judge '{j}' is not a key in judge_settings")

    return True

check_judge_settings staticmethod 

check_judge_settings(judge_settings: dict[str, dict]) -> bool

Method to check if the judge settings dictionary is valid.

Parameters:

Name Type Description Default
judge_settings dict

A dictionary of judge settings with the keys “api”, “model_name”, “parameters”. Used to define the judge LLMs to be used in the judging process

 required

Returns:

Type Description
bool

True if the judge settings dictionary is valid. Errors will be raised if the dictionary is invalid
Source code in src/prompto/judge.py 
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
@staticmethod
def check_judge_settings(judge_settings: dict[str, dict]) -> bool:
    """
    Method to check if the judge settings dictionary is valid.

    Parameters
    ----------
    judge_settings : dict
        A dictionary of judge settings with the keys "api",
        "model_name", "parameters". Used to define the
        judge LLMs to be used in the judging process

    Returns
    -------
    bool
        True if the judge settings dictionary is valid.
        Errors will be raised if the dictionary is invalid
    """
    if not isinstance(judge_settings, dict):
        raise TypeError("judge_settings must be a dictionary")

    for judge, settings in judge_settings.items():
        if not isinstance(settings, dict):
            raise TypeError(f"Value for judge key '{judge}' must be a dictionary")
        if "api" not in settings:
            raise KeyError(f"'api' key not found in settings for judge '{judge}'")
        if "model_name" not in settings:
            raise KeyError(
                f"'model_name' key not found in settings for judge '{judge}'"
            )
        if "parameters" not in settings:
            raise KeyError(
                f"'parameters' key not found in settings for judge '{judge}'"
            )
        if not isinstance(settings["parameters"], dict):
            raise TypeError(
                f"Value for 'parameters' key must be a dictionary for judge '{judge}'"
            )

    return True

create_judge_file 

create_judge_file(judge: list[str] | str, out_filepath: str) -> list[dict]

Method to create a judge file (as a jsonl file) containing the input prompt dictionaries to be processed by the judge LLM(s).

Parameters:

Name Type Description Default
judge list[str] | str

A list of judges or a single judge to be used to. These must be keys in the judge settings dictionary, otherwise a KeyError will be raised

 required
out_filepath str

The path to the output file where the judge inputs will be saved as a jsonl file

 required

Returns:

Type Description
list[dict]

A list of dictionaries containing the input prompt for the judge LLM(s). Each dictionary will contain a new prompt for each prompt/response pair in the completed_responses list using the template_prompt
Source code in src/prompto/judge.py 
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
def create_judge_file(
    self, judge: list[str] | str, out_filepath: str
) -> list[dict]:
    """
    Method to create a judge file (as a jsonl file) containing
    the input prompt dictionaries to be processed by the judge LLM(s).

    Parameters
    ----------
    judge : list[str] | str
        A list of judges or a single judge to be used to.
        These must be keys in the judge settings dictionary,
        otherwise a KeyError will be raised
    out_filepath : str
        The path to the output file where the judge inputs
        will be saved as a jsonl file

    Returns
    -------
    list[dict]
        A list of dictionaries containing the input prompt
        for the judge LLM(s). Each dictionary will contain a
        new prompt for each prompt/response pair in the
        completed_responses list using the template_prompt
    """
    if not out_filepath.endswith(".jsonl"):
        raise ValueError("out_filepath must end with '.jsonl'")

    judge_prompts = self.create_judge_inputs(judge=judge)

    logging.info(f"Creating judge file at {out_filepath}...")
    with open(out_filepath, "w", encoding="utf-8") as f:
        for j_input in tqdm(
            judge_prompts,
            desc=f"Writing judge prompts to {out_filepath}",
            unit="prompts",
        ):
            json.dump(j_input, f)
            f.write("\n")

    return judge_prompts

create_judge_inputs 

create_judge_inputs(judge: list[str] | str) -> list[dict]

Method to create a list of input prompt dictionaries to be processed by the judge LLM(s).

Parameters:

Name Type Description Default
judge list[str] | str

A list of judges or a single judge to be used to. These must be keys in the judge settings dictionary, otherwise a KeyError will be raised

 required

Returns:

Type Description
list[dict]

A list of dictionaries containing the input prompt for the judge LLM(s). Each dictionary will contain a new prompt for each prompt/response pair in the completed_responses list using the template_prompt
Source code in src/prompto/judge.py 
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
def create_judge_inputs(self, judge: list[str] | str) -> list[dict]:
    """
    Method to create a list of input prompt dictionaries to
    be processed by the judge LLM(s).

    Parameters
    ----------
    judge : list[str] | str
        A list of judges or a single judge to be used to.
        These must be keys in the judge settings dictionary,
        otherwise a KeyError will be raised

    Returns
    -------
    list[dict]
        A list of dictionaries containing the input prompt
        for the judge LLM(s). Each dictionary will contain a
        new prompt for each prompt/response pair in the
        completed_responses list using the template_prompt
    """
    if isinstance(judge, str):
        judge = [judge]

    assert self.check_judge_in_judge_settings(
        judge=judge, judge_settings=self.judge_settings
    )

    self.judge_prompts = []
    for j in judge:
        for template_name, template_prompt in self.template_prompts.items():
            self.judge_prompts += [
                {
                    "id": f"judge-{j}-{template_name}-{str(response.get('id', 'NA'))}",
                    "template_name": template_name,
                    "prompt": template_prompt.format(
                        INPUT_PROMPT=response["prompt"],
                        OUTPUT_RESPONSE=response["response"],
                    ),
                }
                | self.judge_settings[j]
                | {f"input-{k}": v for k, v in response.items()}
                for response in tqdm(
                    self.completed_responses,
                    desc=f"Creating judge inputs for judge '{j}' and template '{template_name}'",
                    unit="responses",
                )
            ]

    return self.judge_prompts

load_judge_folder 

load_judge_folder(
    judge_folder: str, templates: str | list[str] = "template.txt"
) -> tuple[dict[str, str], dict]

Parses the judge_folder to load the template prompt string and judge settings dictionary.

The judge_folder should be a path to the judge folder containing the template files and settings.json files.

We read the template from judge_folder/template.txt and the settings from judge_folder/settings.json. If either of these files do not exist, a FileNotFoundError will be raised.

Parameters:

Name Type Description Default
judge_folder str

Path to the judge folder containing the template files and settings.json files

 required
templates str | list[str]

Path(s) to the template file(s) to be used for the judge. By default, this is ‘template.txt’. These files must be in the judge folder and end with ‘.txt’

 'template.txt'

Returns:

Type Description
tuple[dict[str, str], dict]

A tuple containing the template prompt string, which are given as a dictionary with the template name as the key (the template file name without the ‘.txt’ extension) and the value as the template string, and the judge settings dictionary
Source code in src/prompto/judge.py 
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
def load_judge_folder(
    judge_folder: str, templates: str | list[str] = "template.txt"
) -> tuple[dict[str, str], dict]:
    """
    Parses the judge_folder to load the template prompt
    string and judge settings dictionary.

    The judge_folder should be a path to the judge
    folder containing the template files and settings.json files.

    We read the template from judge_folder/template.txt
    and the settings from judge_folder/settings.json. If
    either of these files do not exist, a FileNotFoundError
    will be raised.

    Parameters
    ----------
    judge_folder : str
        Path to the judge folder containing the template files
        and settings.json files
    templates : str | list[str]
        Path(s) to the template file(s) to be used for the judge.
        By default, this is 'template.txt'. These files must be
        in the judge folder and end with '.txt'

    Returns
    -------
    tuple[dict[str, str], dict]
        A tuple containing the template prompt string, which
        are given as a dictionary with the template name as the
        key (the template file name without the '.txt' extension)
        and the value as the template string, and the judge
        settings dictionary
    """
    if not os.path.isdir(judge_folder):
        raise ValueError(
            f"judge folder '{judge_folder}' must be a valid path to a folder"
        )
    if isinstance(templates, str):
        templates = [templates]

    template_prompts = {}
    for template in templates:
        template_path = os.path.join(judge_folder, template)
        if not template_path.endswith(".txt"):
            raise ValueError(f"Template file '{template_path}' must end with '.txt'")

        try:
            with open(template_path, "r", encoding="utf-8") as f:
                template_prompts[template.split(".")[0]] = f.read()
        except FileNotFoundError as exc:
            raise FileNotFoundError(
                f"Template file '{template_path}' does not exist"
            ) from exc

    try:
        judge_settings_path = os.path.join(judge_folder, "settings.json")
        with open(judge_settings_path, "r", encoding="utf-8") as f:
            judge_settings = json.load(f)
    except FileNotFoundError as exc:
        raise FileNotFoundError(
            f"Judge settings file '{judge_settings_path}' does not exist"
        ) from exc

    return template_prompts, judge_settings