Support specifying custom cost per token (All-Hands-AI#2083)

* support specifying custom cost per token

* fix test for new attrs

* add to docs

---------

Co-authored-by: Engel Nyst <enyst@users.noreply.github.com>

opendevin/core/config.py

@@ -44,6 +44,8 @@ class LLMConfig(metaclass=Singleton):
         custom_llm_provider: The custom LLM provider to use. This is undocumented in opendevin, and normally not used. It is documented on the litellm side.
         max_input_tokens: The maximum number of input tokens. Note that this is currently unused, and the value at runtime is actually the total tokens in OpenAI (e.g. 128,000 tokens for GPT-4).
         max_output_tokens: The maximum number of output tokens. This is sent to the LLM.
+        input_cost_per_token: The cost per input token. This will available in logs for the user to check.
+        output_cost_per_token: The cost per output token. This will available in logs for the user to check.
     """
 
     model: str = 'gpt-3.5-turbo'
@@ -66,6 +68,8 @@ class LLMConfig(metaclass=Singleton):
     custom_llm_provider: str | None = None
     max_input_tokens: int | None = None
     max_output_tokens: int | None = None
+    input_cost_per_token: float | None = None
+    output_cost_per_token: float | None = None
 
     def defaults_to_dict(self) -> dict:
         """

opendevin/llm/llm.py

@@ -11,6 +11,7 @@
     RateLimitError,
     ServiceUnavailableError,
 )
+from litellm.types.utils import CostPerToken
 from tenacity import (
     retry,
     retry_if_exception_type,
@@ -267,9 +268,23 @@ def completion_cost(self, response):
         Returns:
             number: The cost of the response.
         """
+        extra_kwargs = {}
+        if (
+            config.llm.input_cost_per_token is not None
+            and config.llm.output_cost_per_token is not None
+        ):
+            cost_per_token = CostPerToken(
+                input_cost_per_token=config.llm.input_cost_per_token,
+                output_cost_per_token=config.llm.output_cost_per_token,
+            )
+            logger.info(f'Using custom cost per token: {cost_per_token}')
+            extra_kwargs['custom_cost_per_token'] = cost_per_token
+
         if not self.is_local():
             try:
-                cost = litellm_completion_cost(completion_response=response)
+                cost = litellm_completion_cost(
+                    completion_response=response, **extra_kwargs
+                )
                 self.metrics.add_cost(cost)
                 return cost
             except Exception:

tests/unit/test_config.py

@@ -232,6 +232,8 @@ def test_api_keys_repr_str():
         'api_key',
         'aws_access_key_id',
         'aws_secret_access_key',
+        'input_cost_per_token',
+        'output_cost_per_token',
     ]
     for attr_name in dir(LLMConfig):
         if (