Skip to content

scorer

includes 

includes(prompt_dict: dict) -> dict

Returns a True if the prompt_dict[“response”] includes the prompt_dict[“expected_response”].

Parameters:

Name Type Description Default
prompt_dict dict

A dictionary containing a “response” and “expected_response” key

 required

Returns:

Type Description
dict

A dictionary containing the “includes” key with a boolean value of the comparison between the “response” and “expected_response” keys.
Source code in src/prompto/scorer.py 
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
def includes(prompt_dict: dict) -> dict:
    """
    Returns a True if the prompt_dict["response"]
    includes the prompt_dict["expected_response"].

    Parameters
    ----------
    prompt_dict : dict
        A dictionary containing a "response" and
        "expected_response" key

    Returns
    -------
    dict
        A dictionary containing the "includes" key with
        a boolean value of the comparison between the
        "response" and "expected_response" keys.
    """
    prompt_dict["includes"] = (
        prompt_dict["expected_response"] in prompt_dict["response"]
    )
    return prompt_dict

match 

match(prompt_dict: dict) -> dict

Returns a True if the prompt_dict[“response”] is equal to the prompt_dict[“expected_response”].

Parameters:

Name Type Description Default
prompt_dict dict

A dictionary containing a “response” and “expected_response” key

 required

Returns:

Type Description
dict

A dictionary containing the “match” key with a boolean value of the comparison between the “response” and “expected_response” keys.
Source code in src/prompto/scorer.py 
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
def match(prompt_dict: dict) -> dict:
    """
    Returns a True if the prompt_dict["response"]
    is equal to the prompt_dict["expected_response"].

    Parameters
    ----------
    prompt_dict : dict
        A dictionary containing a "response" and
        "expected_response" key

    Returns
    -------
    dict
        A dictionary containing the "match" key with
        a boolean value of the comparison between the
        "response" and "expected_response" keys.
    """
    prompt_dict["match"] = prompt_dict["response"] == prompt_dict["expected_response"]
    return prompt_dict

obtain_scoring_functions 

obtain_scoring_functions(
    scorer: str | list[str], scoring_functions_dict: dict[str, Callable]
) -> list[Callable]

Check if the scorer(s) provided are in the scoring_functions_dict.

Parameters:

Name Type Description Default
scorer str | list[str]

A single scorer or a list of scorers to check if they are keys in the scoring_functions_dict dictionary

 required
scoring_functions_dict dict[str, Callable]

A dictionary of scoring functions with the keys as the scorer names and the values as the scoring functions

 required

Returns:

Type Description
list[Callable]

List of scoring functions that correspond to the scorers
Source code in src/prompto/scorer.py 
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
def obtain_scoring_functions(
    scorer: str | list[str], scoring_functions_dict: dict[str, Callable]
) -> list[Callable]:
    """
    Check if the scorer(s) provided are in the scoring_functions_dict.

    Parameters
    ----------
    scorer : str | list[str]
        A single scorer or a list of scorers to check if they
        are keys in the scoring_functions_dict dictionary
    scoring_functions_dict : dict[str, Callable]
        A dictionary of scoring functions with the keys as the
        scorer names and the values as the scoring functions

    Returns
    -------
    list[Callable]
        List of scoring functions that correspond to the scorers
    """
    if isinstance(scorer, str):
        scorer = [scorer]

    functions = []
    for s in scorer:
        if not isinstance(s, str):
            raise TypeError("If scorer is a list, each element must be a string")
        if s not in scoring_functions_dict.keys():
            raise KeyError(
                f"Scorer '{s}' is not a key in scoring_functions_dict. "
                f"Available scorers are: {list(scoring_functions_dict.keys())}"
            )

        functions.append(scoring_functions_dict[s])

    logging.info(f"Scoring functions to be used: {scorer}")
    return functions