Skip to content

Model settings

ModelSettings dataclass

Settings to use when calling an LLM.

This class holds optional model configuration parameters (e.g. temperature, top_p, penalties, truncation, etc.).

Not all models/providers support all of these parameters, so please check the API documentation for the specific model and provider you are using.

Default values: - temperature: 0.7 (can be overridden by CAI_TEMPERATURE env var) - top_p: 1.0 (can be overridden by CAI_TOP_P env var)

Source code in src/cai/sdk/agents/model_settings.py 
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
@dataclass
class ModelSettings:
    """Settings to use when calling an LLM.

    This class holds optional model configuration parameters (e.g. temperature,
    top_p, penalties, truncation, etc.).

    Not all models/providers support all of these parameters, so please check the API documentation
    for the specific model and provider you are using.

    Default values:
    - temperature: 0.7 (can be overridden by CAI_TEMPERATURE env var)
    - top_p: 1.0 (can be overridden by CAI_TOP_P env var)
    """

    temperature: float | None = None
    """The temperature to use when calling the model. Default: 0.7 (from CAI_TEMPERATURE env var)."""

    top_p: float | None = None
    """The top_p to use when calling the model. Default: 1.0 (from CAI_TOP_P env var)."""

    frequency_penalty: float | None = None
    """The frequency penalty to use when calling the model."""

    presence_penalty: float | None = None
    """The presence penalty to use when calling the model."""

    tool_choice: Literal["auto", "required", "none"] | str | None = None
    """The tool choice to use when calling the model."""

    parallel_tool_calls: bool | None = None
    """Whether to use parallel tool calls when calling the model.
    Defaults to False if not provided."""

    truncation: Literal["auto", "disabled"] | None = None
    """The truncation strategy to use when calling the model."""

    max_tokens: int | None = None
    """The maximum number of output tokens to generate."""

    store: bool | None = None
    """Whether to store the generated model response for later retrieval.
    Defaults to True if not provided."""

    agent_model: str | None = None
    """The model from the Agent class. If set, this will override the model provided
    to the OpenAIChatCompletionsModel during initialization."""

    def resolve(self, override: ModelSettings | None) -> ModelSettings:
        """Produce a new ModelSettings by overlaying any non-None values from the
        override on top of this instance."""
        if override is None:
            return self

        changes = {
            field.name: getattr(override, field.name)
            for field in fields(self)
            if getattr(override, field.name) is not None
        }
        return replace(self, **changes)

    def with_defaults(self) -> ModelSettings:
        """Return a new ModelSettings with defaults applied for None values.

        Uses CAI_TEMPERATURE env var (default 0.7) and CAI_TOP_P env var (default 1.0).
        """
        return replace(
            self,
            temperature=self.temperature if self.temperature is not None else get_default_temperature(),
            top_p=self.top_p if self.top_p is not None else get_default_top_p(),
        )

    @classmethod
    def from_env(cls) -> ModelSettings:
        """Create ModelSettings with values from environment variables.

        Reads CAI_TEMPERATURE (default 0.7) and CAI_TOP_P (default 1.0).
        """
        return cls(
            temperature=get_default_temperature(),
            top_p=get_default_top_p(),
        )

temperature class-attribute instance-attribute 

temperature: float | None = None

The temperature to use when calling the model. Default: 0.7 (from CAI_TEMPERATURE env var).

top_p class-attribute instance-attribute 

top_p: float | None = None

The top_p to use when calling the model. Default: 1.0 (from CAI_TOP_P env var).

frequency_penalty class-attribute instance-attribute 

frequency_penalty: float | None = None

The frequency penalty to use when calling the model.

presence_penalty class-attribute instance-attribute 

presence_penalty: float | None = None

The presence penalty to use when calling the model.

tool_choice class-attribute instance-attribute 

tool_choice: (
    Literal["auto", "required", "none"] | str | None
) = None

The tool choice to use when calling the model.

parallel_tool_calls class-attribute instance-attribute 

parallel_tool_calls: bool | None = None

Whether to use parallel tool calls when calling the model. Defaults to False if not provided.

truncation class-attribute instance-attribute 

truncation: Literal['auto', 'disabled'] | None = None

The truncation strategy to use when calling the model.

max_tokens class-attribute instance-attribute 

max_tokens: int | None = None

The maximum number of output tokens to generate.

store class-attribute instance-attribute 

store: bool | None = None

Whether to store the generated model response for later retrieval. Defaults to True if not provided.

agent_model class-attribute instance-attribute 

agent_model: str | None = None

The model from the Agent class. If set, this will override the model provided to the OpenAIChatCompletionsModel during initialization.

resolve 

resolve(override: ModelSettings | None) -> ModelSettings

Produce a new ModelSettings by overlaying any non-None values from the override on top of this instance.

Source code in src/cai/sdk/agents/model_settings.py 
76
77
78
79
80
81
82
83
84
85
86
87
def resolve(self, override: ModelSettings | None) -> ModelSettings:
    """Produce a new ModelSettings by overlaying any non-None values from the
    override on top of this instance."""
    if override is None:
        return self

    changes = {
        field.name: getattr(override, field.name)
        for field in fields(self)
        if getattr(override, field.name) is not None
    }
    return replace(self, **changes)

with_defaults 

with_defaults() -> ModelSettings

Return a new ModelSettings with defaults applied for None values.

Uses CAI_TEMPERATURE env var (default 0.7) and CAI_TOP_P env var (default 1.0).

Source code in src/cai/sdk/agents/model_settings.py 
89
90
91
92
93
94
95
96
97
98
def with_defaults(self) -> ModelSettings:
    """Return a new ModelSettings with defaults applied for None values.

    Uses CAI_TEMPERATURE env var (default 0.7) and CAI_TOP_P env var (default 1.0).
    """
    return replace(
        self,
        temperature=self.temperature if self.temperature is not None else get_default_temperature(),
        top_p=self.top_p if self.top_p is not None else get_default_top_p(),
    )

from_env classmethod 

from_env() -> ModelSettings

Create ModelSettings with values from environment variables.

Reads CAI_TEMPERATURE (default 0.7) and CAI_TOP_P (default 1.0).

Source code in src/cai/sdk/agents/model_settings.py 
100
101
102
103
104
105
106
107
108
109
@classmethod
def from_env(cls) -> ModelSettings:
    """Create ModelSettings with values from environment variables.

    Reads CAI_TEMPERATURE (default 0.7) and CAI_TOP_P (default 1.0).
    """
    return cls(
        temperature=get_default_temperature(),
        top_p=get_default_top_p(),
    )

get_default_temperature 

get_default_temperature() -> float

Get the default temperature from environment or use DEFAULT_TEMPERATURE (0.7).

Source code in src/cai/sdk/agents/model_settings.py 
12
13
14
15
16
17
def get_default_temperature() -> float:
    """Get the default temperature from environment or use DEFAULT_TEMPERATURE (0.7)."""
    try:
        return float(os.getenv("CAI_TEMPERATURE", str(DEFAULT_TEMPERATURE)))
    except (ValueError, TypeError):
        return DEFAULT_TEMPERATURE

get_default_top_p 

get_default_top_p() -> float

Get the default top_p from environment or use DEFAULT_TOP_P (1.0).

Source code in src/cai/sdk/agents/model_settings.py 
20
21
22
23
24
25
def get_default_top_p() -> float:
    """Get the default top_p from environment or use DEFAULT_TOP_P (1.0)."""
    try:
        return float(os.getenv("CAI_TOP_P", str(DEFAULT_TOP_P)))
    except (ValueError, TypeError):
        return DEFAULT_TOP_P