././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1785864218.5851147 llm-0.32/0000755000175100017510000000000015234420033011664 5ustar00runnerrunner././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/LICENSE0000644000175100017510000002613515234420023012677 0ustar00runnerrunner Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/MANIFEST.in0000644000175100017510000000002715234420023013420 0ustar00runnerrunnerglobal-exclude tests/* ././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1785864218.585059 llm-0.32/PKG-INFO0000644000175100017510000010255315234420033012767 0ustar00runnerrunnerMetadata-Version: 2.4 Name: llm Version: 0.32 Summary: CLI utility and Python library for interacting with Large Language Models from organizations like OpenAI, Anthropic and Gemini plus local models installed on your own machine. Author: Simon Willison License-Expression: Apache-2.0 Project-URL: Homepage, https://github.com/simonw/llm Project-URL: Documentation, https://llm.datasette.io/ Project-URL: Issues, https://github.com/simonw/llm/issues Project-URL: CI, https://github.com/simonw/llm/actions Project-URL: Changelog, https://github.com/simonw/llm/releases Classifier: Development Status :: 4 - Beta Classifier: Intended Audience :: Developers Classifier: Intended Audience :: End Users/Desktop Classifier: Intended Audience :: Science/Research Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: 3.13 Classifier: Programming Language :: Python :: 3.14 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence Classifier: Topic :: Text Processing :: Linguistic Classifier: Topic :: Utilities Requires-Python: >=3.10 Description-Content-Type: text/markdown License-File: LICENSE Requires-Dist: click Requires-Dist: condense-json>=1.1 Requires-Dist: openai>=2.32.0 Requires-Dist: click-default-group>=1.2.3 Requires-Dist: sqlite-utils>=4.0 Requires-Dist: pydantic>=2.0.0 Requires-Dist: PyYAML Requires-Dist: pluggy Requires-Dist: python-ulid Requires-Dist: setuptools Requires-Dist: pip Requires-Dist: pyreadline3; sys_platform == "win32" Requires-Dist: puremagic Dynamic: license-file # LLM [![GitHub repo](https://img.shields.io/badge/github-repo-green)](https://github.com/simonw/llm) [![PyPI](https://img.shields.io/pypi/v/llm.svg)](https://pypi.org/project/llm/) [![Changelog](https://img.shields.io/github/v/release/simonw/llm?include_prereleases&label=changelog)](https://llm.datasette.io/en/stable/changelog.html) [![Tests](https://github.com/simonw/llm/workflows/Test/badge.svg)](https://github.com/simonw/llm/actions?query=workflow%3ATest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/llm/blob/main/LICENSE) [![Discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://datasette.io/discord-llm) [![Homebrew](https://img.shields.io/homebrew/installs/dy/llm?color=yellow&label=homebrew&logo=homebrew)](https://formulae.brew.sh/formula/llm) A CLI tool and Python library for interacting with **OpenAI**, **Anthropic’s Claude**, **Google’s Gemini**, **Meta’s Llama** and dozens of other Large Language Models, both via remote APIs and with models that can be installed and run on your own machine. Watch **[Language models on the command-line](https://www.youtube.com/watch?v=QUXQNi6jQ30)** on YouTube for a demo or [read the accompanying detailed notes](https://simonwillison.net/2024/Jun/17/cli-language-models/). With LLM you can: - [Run prompts from the command-line](https://llm.datasette.io/en/stable/usage.html#usage-executing-prompts) - [Store prompts and responses in SQLite](https://llm.datasette.io/en/stable/logging.html#logging) - [Generate and store embeddings](https://llm.datasette.io/en/stable/embeddings/index.html#embeddings) - [Extract structured content from text and images](https://llm.datasette.io/en/stable/schemas.html#schemas) - [Grant models the ability to execute tools](https://llm.datasette.io/en/stable/tools.html#tools) - … and much, much more ## Quick start First, install LLM using `pip` or Homebrew or `pipx` or `uv`: ```bash pip install llm ``` Or with Homebrew (see [warning note](https://llm.datasette.io/en/stable/setup.html#homebrew-warning)): ```bash brew install llm ``` Or with [pipx](https://pypa.github.io/pipx/): ```bash pipx install llm ``` Or with [uv](https://docs.astral.sh/uv/guides/tools/) ```bash uv tool install llm ``` Use LLM to run prompts or start chats against an arbitrary OpenAI-compatible Chat Completions endpoint, such as [LM Studio](https://lmstudio.ai). With `uvx`, you can do this without installing LLM first: ```bash uvx llm openai endpoint http://localhost:1234/v1 \ -m google/gemma-4-12b \ "What is the capital of France?" uvx llm openai endpoint http://localhost:1234/v1 \ -m google/gemma-4-12b \ --chat ``` Add `--key your-api-key` if the endpoint requires authentication. See [Run against an endpoint without configuring it](https://llm.datasette.io/en/stable/other-models.html#openai-endpoint) for more options. If you have an [OpenAI API key](https://platform.openai.com/api-keys) key you can run this: ```bash # Paste your OpenAI API key into this llm keys set openai # Run a prompt (with the default gpt-5.6-luna model) llm "Ten fun names for a pet pelican" # Extract text from an image llm "extract text" -a scanned-document.jpg # Use a system prompt against a file cat myfile.py | llm -s "Explain this code" ``` Run prompts against [Gemini](https://aistudio.google.com/apikey) or [Anthropic](https://console.anthropic.com/) with their respective plugins: ```bash llm install llm-gemini llm keys set gemini # Paste Gemini API key here llm -m gemini-3.5-flash 'Tell me fun facts about Mountain View' llm install llm-anthropic llm keys set anthropic # Paste Anthropic API key here llm -m claude-sonnet-5 'Impress me with wild facts about turnips' ``` You can also [install a plugin](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#installing-plugins) to access models that can run on your local device. If you use [Ollama](https://ollama.com/): ```bash # Install the plugin llm install llm-ollama # Download and run a prompt against the Orca Mini 7B model ollama pull llama3.2:latest llm -m llama3.2:latest 'What is the capital of France?' ``` To start [an interactive chat](https://llm.datasette.io/en/stable/usage.html#usage-chat) with a model, use `llm chat`: ```bash llm chat -m gpt-4.1 ``` ```default Chatting with gpt-4.1 Type 'exit' or 'quit' to exit Type '!multi' to enter multiple lines, then '!end' to finish Type '!edit' to open your default editor and modify the prompt. Type '!fragment [ ...]' to insert one or more fragments > Tell me a joke about a pelican Why don't pelicans like to tip waiters? Because they always have a big bill! ``` ## Project news - 29th April 2026: [LLM 0.32a0 is a major backwards-compatible refactor](https://simonwillison.net/2026/Apr/29/llm/) - 11th August 2025: [LLM 0.27, the annotated release notes: GPT-5 and improved tool calling](https://simonwillison.net/2025/Aug/11/llm-027/) - 27th May 2025: [Large Language Models can run tools in your terminal with LLM 0.26](https://simonwillison.net/2025/May/27/llm-tools/) - 5th May 2025: [Feed a video to a vision LLM as a sequence of JPEG frames on the CLI (also LLM 0.25)](https://simonwillison.net/2025/May/5/llm-video-frames/) - 7th April 2025: [Long context support in LLM 0.24 using fragments and template plugins](https://simonwillison.net/2025/Apr/7/long-context-llm/) - 28th February 2025: [Structured data extraction from unstructured content using LLM schemas](https://simonwillison.net/2025/Feb/28/llm-schemas/) - 17th February 2025: [LLM 0.22, the annotated release notes](https://simonwillison.net/2025/Feb/17/llm/) - 29th October 2024: [You can now run prompts against images, audio and video in your terminal using LLM](https://simonwillison.net/2024/Oct/29/llm-multi-modal/) - 26th January 2024: [LLM 0.13: The annotated release notes](https://simonwillison.net/2024/Jan/26/llm/) - 12th September 2023: [Build an image search engine with llm-clip, chat with models with llm chat](https://simonwillison.net/2023/Sep/12/llm-clip-and-chat/) - 4th September 2023: [LLM now provides tools for working with embeddings](https://simonwillison.net/2023/Sep/4/llm-embeddings/) - 12th July 2023: [The LLM CLI tool now supports self-hosted language models via plugins](https://simonwillison.net/2023/Jul/12/llm/) - 18th May 2023: [llm, ttok and strip-tags—CLI tools for working with ChatGPT and other LLMs](https://simonwillison.net/2023/May/18/cli-tools-for-llms/) - 4th April 2023: [The original announcement of the llm CLI tool](https://simonwillison.net/2023/Apr/4/llm/) For everything else, see [the llm tag](https://simonwillison.net/tags/llm/) on my blog. ## Contents * [Setup](https://llm.datasette.io/en/stable/setup.html) * [Installation](https://llm.datasette.io/en/stable/setup.html#installation) * [Upgrading to the latest version](https://llm.datasette.io/en/stable/setup.html#upgrading-to-the-latest-version) * [Using uvx](https://llm.datasette.io/en/stable/setup.html#using-uvx) * [A note about Homebrew and PyTorch](https://llm.datasette.io/en/stable/setup.html#a-note-about-homebrew-and-pytorch) * [Installing plugins](https://llm.datasette.io/en/stable/setup.html#installing-plugins) * [API key management](https://llm.datasette.io/en/stable/setup.html#api-key-management) * [Saving and using stored keys](https://llm.datasette.io/en/stable/setup.html#saving-and-using-stored-keys) * [Passing keys using the –key option](https://llm.datasette.io/en/stable/setup.html#passing-keys-using-the-key-option) * [Keys in environment variables](https://llm.datasette.io/en/stable/setup.html#keys-in-environment-variables) * [Configuration](https://llm.datasette.io/en/stable/setup.html#configuration) * [Setting a custom default model](https://llm.datasette.io/en/stable/setup.html#setting-a-custom-default-model) * [Setting a custom directory location](https://llm.datasette.io/en/stable/setup.html#setting-a-custom-directory-location) * [Turning SQLite logging on and off](https://llm.datasette.io/en/stable/setup.html#turning-sqlite-logging-on-and-off) * [Usage](https://llm.datasette.io/en/stable/usage.html) * [Executing a prompt](https://llm.datasette.io/en/stable/usage.html#executing-a-prompt) * [Model options](https://llm.datasette.io/en/stable/usage.html#model-options) * [Attachments](https://llm.datasette.io/en/stable/usage.html#attachments) * [System prompts](https://llm.datasette.io/en/stable/usage.html#system-prompts) * [Tools](https://llm.datasette.io/en/stable/usage.html#tools) * [Extracting fenced code blocks](https://llm.datasette.io/en/stable/usage.html#extracting-fenced-code-blocks) * [JSON output](https://llm.datasette.io/en/stable/usage.html#json-output) * [Schemas](https://llm.datasette.io/en/stable/usage.html#schemas) * [Fragments](https://llm.datasette.io/en/stable/usage.html#fragments) * [Continuing a conversation](https://llm.datasette.io/en/stable/usage.html#continuing-a-conversation) * [Tips for using LLM with Bash or Zsh](https://llm.datasette.io/en/stable/usage.html#tips-for-using-llm-with-bash-or-zsh) * [Completion prompts](https://llm.datasette.io/en/stable/usage.html#completion-prompts) * [Starting an interactive chat](https://llm.datasette.io/en/stable/usage.html#starting-an-interactive-chat) * [Listing available models](https://llm.datasette.io/en/stable/usage.html#listing-available-models) * [Setting default options for models](https://llm.datasette.io/en/stable/usage.html#setting-default-options-for-models) * [OpenAI models](https://llm.datasette.io/en/stable/openai-models.html) * [Configuration](https://llm.datasette.io/en/stable/openai-models.html#configuration) * [OpenAI language models](https://llm.datasette.io/en/stable/openai-models.html#openai-language-models) * [Model features](https://llm.datasette.io/en/stable/openai-models.html#model-features) * [Web Search](https://llm.datasette.io/en/stable/openai-models.html#web-search) * [Code Interpreter](https://llm.datasette.io/en/stable/openai-models.html#code-interpreter) * [Fast mode and service tiers](https://llm.datasette.io/en/stable/openai-models.html#fast-mode-and-service-tiers) * [OpenAI embedding models](https://llm.datasette.io/en/stable/openai-models.html#openai-embedding-models) * [OpenAI completion models](https://llm.datasette.io/en/stable/openai-models.html#openai-completion-models) * [Adding more OpenAI models](https://llm.datasette.io/en/stable/openai-models.html#adding-more-openai-models) * [Other models](https://llm.datasette.io/en/stable/other-models.html) * [Installing and using a local model](https://llm.datasette.io/en/stable/other-models.html#installing-and-using-a-local-model) * [OpenAI-compatible models](https://llm.datasette.io/en/stable/other-models.html#openai-compatible-models) * [Run against an endpoint without configuring it](https://llm.datasette.io/en/stable/other-models.html#run-against-an-endpoint-without-configuring-it) * [Configure an OpenAI-compatible model](https://llm.datasette.io/en/stable/other-models.html#configure-an-openai-compatible-model) * [Extra HTTP headers](https://llm.datasette.io/en/stable/other-models.html#extra-http-headers) * [Tools](https://llm.datasette.io/en/stable/tools.html) * [How tools work](https://llm.datasette.io/en/stable/tools.html#how-tools-work) * [Trying out tools](https://llm.datasette.io/en/stable/tools.html#trying-out-tools) * [LLM’s implementation of tools](https://llm.datasette.io/en/stable/tools.html#llm-s-implementation-of-tools) * [Default tools](https://llm.datasette.io/en/stable/tools.html#default-tools) * [Tips for implementing tools](https://llm.datasette.io/en/stable/tools.html#tips-for-implementing-tools) * [Schemas](https://llm.datasette.io/en/stable/schemas.html) * [Schemas tutorial](https://llm.datasette.io/en/stable/schemas.html#schemas-tutorial) * [Getting started with dogs](https://llm.datasette.io/en/stable/schemas.html#getting-started-with-dogs) * [Extracting people from a news articles](https://llm.datasette.io/en/stable/schemas.html#extracting-people-from-a-news-articles) * [Using JSON schemas](https://llm.datasette.io/en/stable/schemas.html#using-json-schemas) * [Ways to specify a schema](https://llm.datasette.io/en/stable/schemas.html#ways-to-specify-a-schema) * [Concise LLM schema syntax](https://llm.datasette.io/en/stable/schemas.html#concise-llm-schema-syntax) * [Saving reusable schemas in templates](https://llm.datasette.io/en/stable/schemas.html#saving-reusable-schemas-in-templates) * [Browsing logged JSON objects created using schemas](https://llm.datasette.io/en/stable/schemas.html#browsing-logged-json-objects-created-using-schemas) * [Templates](https://llm.datasette.io/en/stable/templates.html) * [Getting started with –save](https://llm.datasette.io/en/stable/templates.html#getting-started-with-save) * [Using a template](https://llm.datasette.io/en/stable/templates.html#using-a-template) * [Listing available templates](https://llm.datasette.io/en/stable/templates.html#listing-available-templates) * [Templates as YAML files](https://llm.datasette.io/en/stable/templates.html#templates-as-yaml-files) * [System prompts](https://llm.datasette.io/en/stable/templates.html#system-prompts) * [Fragments](https://llm.datasette.io/en/stable/templates.html#fragments) * [Options](https://llm.datasette.io/en/stable/templates.html#options) * [Tools](https://llm.datasette.io/en/stable/templates.html#tools) * [Schemas](https://llm.datasette.io/en/stable/templates.html#schemas) * [Additional template variables](https://llm.datasette.io/en/stable/templates.html#additional-template-variables) * [Specifying default parameters](https://llm.datasette.io/en/stable/templates.html#specifying-default-parameters) * [Configuring code extraction](https://llm.datasette.io/en/stable/templates.html#configuring-code-extraction) * [Setting a default model for a template](https://llm.datasette.io/en/stable/templates.html#setting-a-default-model-for-a-template) * [Template loaders from plugins](https://llm.datasette.io/en/stable/templates.html#template-loaders-from-plugins) * [Fragments](https://llm.datasette.io/en/stable/fragments.html) * [Using fragments in a prompt](https://llm.datasette.io/en/stable/fragments.html#using-fragments-in-a-prompt) * [Using fragments in chat](https://llm.datasette.io/en/stable/fragments.html#using-fragments-in-chat) * [Browsing fragments](https://llm.datasette.io/en/stable/fragments.html#browsing-fragments) * [Setting aliases for fragments](https://llm.datasette.io/en/stable/fragments.html#setting-aliases-for-fragments) * [Viewing fragments in your logs](https://llm.datasette.io/en/stable/fragments.html#viewing-fragments-in-your-logs) * [Using fragments from plugins](https://llm.datasette.io/en/stable/fragments.html#using-fragments-from-plugins) * [Listing available fragment prefixes](https://llm.datasette.io/en/stable/fragments.html#listing-available-fragment-prefixes) * [Model aliases](https://llm.datasette.io/en/stable/aliases.html) * [Listing aliases](https://llm.datasette.io/en/stable/aliases.html#listing-aliases) * [Adding a new alias](https://llm.datasette.io/en/stable/aliases.html#adding-a-new-alias) * [Removing an alias](https://llm.datasette.io/en/stable/aliases.html#removing-an-alias) * [Viewing the aliases file](https://llm.datasette.io/en/stable/aliases.html#viewing-the-aliases-file) * [Embeddings](https://llm.datasette.io/en/stable/embeddings/index.html) * [Embedding with the CLI](https://llm.datasette.io/en/stable/embeddings/cli.html) * [llm embed](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed) * [llm embed-multi](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed-multi) * [llm similar](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-similar) * [llm embed-models](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed-models) * [llm collections list](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-collections-list) * [llm collections delete](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-collections-delete) * [Using embeddings from Python](https://llm.datasette.io/en/stable/embeddings/python-api.html) * [Working with collections](https://llm.datasette.io/en/stable/embeddings/python-api.html#working-with-collections) * [Retrieving similar items](https://llm.datasette.io/en/stable/embeddings/python-api.html#retrieving-similar-items) * [SQL schema](https://llm.datasette.io/en/stable/embeddings/python-api.html#sql-schema) * [Writing plugins to add new embedding models](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html) * [`EmbeddingModel`](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html#llm.EmbeddingModel) * [Embedding binary content](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html#embedding-binary-content) * [Embedding storage format](https://llm.datasette.io/en/stable/embeddings/storage.html) * [Plugins](https://llm.datasette.io/en/stable/plugins/index.html) * [Installing plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html) * [Listing installed plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#listing-installed-plugins) * [Running with a subset of plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#running-with-a-subset-of-plugins) * [Plugin directory](https://llm.datasette.io/en/stable/plugins/directory.html) * [Local models](https://llm.datasette.io/en/stable/plugins/directory.html#local-models) * [Remote APIs](https://llm.datasette.io/en/stable/plugins/directory.html#remote-apis) * [Tools](https://llm.datasette.io/en/stable/plugins/directory.html#tools) * [Fragments and template loaders](https://llm.datasette.io/en/stable/plugins/directory.html#fragments-and-template-loaders) * [Embedding models](https://llm.datasette.io/en/stable/plugins/directory.html#embedding-models) * [Extra commands](https://llm.datasette.io/en/stable/plugins/directory.html#extra-commands) * [Just for fun](https://llm.datasette.io/en/stable/plugins/directory.html#just-for-fun) * [Plugin hooks](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html) * [register_commands(cli)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-commands-cli) * [register_models(register, model_aliases)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-models-register-model-aliases) * [register_embedding_models(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-embedding-models-register) * [register_tools(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-tools-register) * [register_template_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-template-loaders-register) * [register_fragment_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-fragment-loaders-register) * [Developing a model plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html) * [The initial structure of the plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#the-initial-structure-of-the-plugin) * [Installing your plugin to try it out](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#installing-your-plugin-to-try-it-out) * [Building the Markov chain](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#building-the-markov-chain) * [Executing the Markov chain](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#executing-the-markov-chain) * [Adding that to the plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-that-to-the-plugin) * [Understanding execute()](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#understanding-execute) * [Prompts and responses are logged to the database](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#prompts-and-responses-are-logged-to-the-database) * [Adding options](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-options) * [Distributing your plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#distributing-your-plugin) * [GitHub repositories](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#github-repositories) * [Publishing plugins to PyPI](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#publishing-plugins-to-pypi) * [Adding metadata](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-metadata) * [What to do if it breaks](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#what-to-do-if-it-breaks) * [Advanced model plugins](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html) * [Tip: lazily load expensive dependencies](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tip-lazily-load-expensive-dependencies) * [Models that accept API keys](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#models-that-accept-api-keys) * [Async models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#async-models) * [Supporting schemas](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-schemas) * [Supporting tools](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-tools) * [Supporting server-side tools](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-server-side-tools) * [Attachments for multi-modal models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#attachments-for-multi-modal-models) * [Structured messages and streaming events](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#structured-messages-and-streaming-events) * [Condensing logged payloads with json_replacements](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#condensing-logged-payloads-with-json-replacements) * [Consuming prompt.messages in build_messages](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#consuming-prompt-messages-in-build-messages) * [Restoring opaque metadata on subsequent requests](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#restoring-opaque-metadata-on-subsequent-requests) * [Tracking token usage](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-token-usage) * [Tracking resolved model names](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-resolved-model-names) * [LLM_RAISE_ERRORS](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#llm-raise-errors) * [Utility functions for plugins](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html) * [llm.get_key()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-get-key) * [llm.user_dir()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-user-dir) * [llm.ModelError](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-modelerror) * [Response.fake()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#response-fake) * [Python API](https://llm.datasette.io/en/stable/python-api.html) * [Basic prompt execution](https://llm.datasette.io/en/stable/python-api.html#basic-prompt-execution) * [System prompts](https://llm.datasette.io/en/stable/python-api.html#system-prompts) * [Attachments](https://llm.datasette.io/en/stable/python-api.html#attachments) * [Tools](https://llm.datasette.io/en/stable/python-api.html#tools) * [Schemas](https://llm.datasette.io/en/stable/python-api.html#schemas) * [Fragments](https://llm.datasette.io/en/stable/python-api.html#fragments) * [Model options](https://llm.datasette.io/en/stable/python-api.html#model-options) * [Passing an API key](https://llm.datasette.io/en/stable/python-api.html#passing-an-api-key) * [Models from plugins](https://llm.datasette.io/en/stable/python-api.html#models-from-plugins) * [Accessing the underlying JSON](https://llm.datasette.io/en/stable/python-api.html#accessing-the-underlying-json) * [Token usage](https://llm.datasette.io/en/stable/python-api.html#token-usage) * [Streaming responses](https://llm.datasette.io/en/stable/python-api.html#streaming-responses) * [Structured messages and streaming events](https://llm.datasette.io/en/stable/python-api.html#structured-messages-and-streaming-events) * [Async models](https://llm.datasette.io/en/stable/python-api.html#async-models) * [`AsyncResponse`](https://llm.datasette.io/en/stable/python-api.html#llm.AsyncResponse) * [Tool functions can be sync or async](https://llm.datasette.io/en/stable/python-api.html#tool-functions-can-be-sync-or-async) * [Tool use for async models](https://llm.datasette.io/en/stable/python-api.html#tool-use-for-async-models) * [Conversations](https://llm.datasette.io/en/stable/python-api.html#conversations) * [Conversations using tools](https://llm.datasette.io/en/stable/python-api.html#conversations-using-tools) * [Listing models](https://llm.datasette.io/en/stable/python-api.html#listing-models) * [Running code when a response has completed](https://llm.datasette.io/en/stable/python-api.html#running-code-when-a-response-has-completed) * [Other functions](https://llm.datasette.io/en/stable/python-api.html#other-functions) * [set_alias(alias, model_id)](https://llm.datasette.io/en/stable/python-api.html#set-alias-alias-model-id) * [remove_alias(alias)](https://llm.datasette.io/en/stable/python-api.html#remove-alias-alias) * [set_default_model(alias)](https://llm.datasette.io/en/stable/python-api.html#set-default-model-alias) * [get_default_model()](https://llm.datasette.io/en/stable/python-api.html#get-default-model) * [set_default_embedding_model(alias) and get_default_embedding_model()](https://llm.datasette.io/en/stable/python-api.html#set-default-embedding-model-alias-and-get-default-embedding-model) * [Logging to SQLite](https://llm.datasette.io/en/stable/logging.html) * [Viewing the logs](https://llm.datasette.io/en/stable/logging.html#viewing-the-logs) * [-s/–short mode](https://llm.datasette.io/en/stable/logging.html#s-short-mode) * [Logs for a conversation](https://llm.datasette.io/en/stable/logging.html#logs-for-a-conversation) * [Searching the logs](https://llm.datasette.io/en/stable/logging.html#searching-the-logs) * [Filtering past a specific ID](https://llm.datasette.io/en/stable/logging.html#filtering-past-a-specific-id) * [Filtering by model](https://llm.datasette.io/en/stable/logging.html#filtering-by-model) * [Filtering by prompts that used specific fragments](https://llm.datasette.io/en/stable/logging.html#filtering-by-prompts-that-used-specific-fragments) * [Filtering by prompts that used specific tools](https://llm.datasette.io/en/stable/logging.html#filtering-by-prompts-that-used-specific-tools) * [Browsing data collected using schemas](https://llm.datasette.io/en/stable/logging.html#browsing-data-collected-using-schemas) * [Browsing logs using Datasette](https://llm.datasette.io/en/stable/logging.html#browsing-logs-using-datasette) * [Backing up your database](https://llm.datasette.io/en/stable/logging.html#backing-up-your-database) * [The message store](https://llm.datasette.io/en/stable/logging.html#the-message-store) * [Threads, turns, messages and parts](https://llm.datasette.io/en/stable/logging.html#threads-turns-messages-and-parts) * [A worked example](https://llm.datasette.io/en/stable/logging.html#a-worked-example) * [Content addressing as a contract](https://llm.datasette.io/en/stable/logging.html#content-addressing-as-a-contract) * [Forking and shared history](https://llm.datasette.io/en/stable/logging.html#forking-and-shared-history) * [Storage by reference](https://llm.datasette.io/en/stable/logging.html#storage-by-reference) * [The raw provider payload](https://llm.datasette.io/en/stable/logging.html#the-raw-provider-payload) * [Table by table](https://llm.datasette.io/en/stable/logging.html#table-by-table) * [Querying the message store](https://llm.datasette.io/en/stable/logging.html#querying-the-message-store) * [Logging from Python](https://llm.datasette.io/en/stable/logging.html#logging-from-python) * [SQL schema](https://llm.datasette.io/en/stable/logging.html#sql-schema) * [Related tools](https://llm.datasette.io/en/stable/related-tools.html) * [strip-tags](https://llm.datasette.io/en/stable/related-tools.html#strip-tags) * [ttok](https://llm.datasette.io/en/stable/related-tools.html#ttok) * [Symbex](https://llm.datasette.io/en/stable/related-tools.html#symbex) * [CLI reference](https://llm.datasette.io/en/stable/help.html) * [llm –help](https://llm.datasette.io/en/stable/help.html#llm-help) * [llm prompt –help](https://llm.datasette.io/en/stable/help.html#llm-prompt-help) * [llm chat –help](https://llm.datasette.io/en/stable/help.html#llm-chat-help) * [llm keys –help](https://llm.datasette.io/en/stable/help.html#llm-keys-help) * [llm logs –help](https://llm.datasette.io/en/stable/help.html#llm-logs-help) * [llm models –help](https://llm.datasette.io/en/stable/help.html#llm-models-help) * [llm templates –help](https://llm.datasette.io/en/stable/help.html#llm-templates-help) * [llm schemas –help](https://llm.datasette.io/en/stable/help.html#llm-schemas-help) * [llm tools –help](https://llm.datasette.io/en/stable/help.html#llm-tools-help) * [llm aliases –help](https://llm.datasette.io/en/stable/help.html#llm-aliases-help) * [llm fragments –help](https://llm.datasette.io/en/stable/help.html#llm-fragments-help) * [llm plugins –help](https://llm.datasette.io/en/stable/help.html#llm-plugins-help) * [llm install –help](https://llm.datasette.io/en/stable/help.html#llm-install-help) * [llm uninstall –help](https://llm.datasette.io/en/stable/help.html#llm-uninstall-help) * [llm embed –help](https://llm.datasette.io/en/stable/help.html#llm-embed-help) * [llm embed-multi –help](https://llm.datasette.io/en/stable/help.html#llm-embed-multi-help) * [llm similar –help](https://llm.datasette.io/en/stable/help.html#llm-similar-help) * [llm embed-models –help](https://llm.datasette.io/en/stable/help.html#llm-embed-models-help) * [llm collections –help](https://llm.datasette.io/en/stable/help.html#llm-collections-help) * [llm openai –help](https://llm.datasette.io/en/stable/help.html#llm-openai-help) * [Contributing](https://llm.datasette.io/en/stable/contributing.html) * [Updating recorded HTTP API interactions and associated snapshots](https://llm.datasette.io/en/stable/contributing.html#updating-recorded-http-api-interactions-and-associated-snapshots) * [Debugging tricks](https://llm.datasette.io/en/stable/contributing.html#debugging-tricks) * [Documentation](https://llm.datasette.io/en/stable/contributing.html#documentation) * [Release process](https://llm.datasette.io/en/stable/contributing.html#release-process) * [Changelog](https://llm.datasette.io/en/stable/changelog.html) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/README.md0000644000175100017510000007730215234420023013153 0ustar00runnerrunner # LLM [![GitHub repo](https://img.shields.io/badge/github-repo-green)](https://github.com/simonw/llm) [![PyPI](https://img.shields.io/pypi/v/llm.svg)](https://pypi.org/project/llm/) [![Changelog](https://img.shields.io/github/v/release/simonw/llm?include_prereleases&label=changelog)](https://llm.datasette.io/en/stable/changelog.html) [![Tests](https://github.com/simonw/llm/workflows/Test/badge.svg)](https://github.com/simonw/llm/actions?query=workflow%3ATest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/llm/blob/main/LICENSE) [![Discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://datasette.io/discord-llm) [![Homebrew](https://img.shields.io/homebrew/installs/dy/llm?color=yellow&label=homebrew&logo=homebrew)](https://formulae.brew.sh/formula/llm) A CLI tool and Python library for interacting with **OpenAI**, **Anthropic’s Claude**, **Google’s Gemini**, **Meta’s Llama** and dozens of other Large Language Models, both via remote APIs and with models that can be installed and run on your own machine. Watch **[Language models on the command-line](https://www.youtube.com/watch?v=QUXQNi6jQ30)** on YouTube for a demo or [read the accompanying detailed notes](https://simonwillison.net/2024/Jun/17/cli-language-models/). With LLM you can: - [Run prompts from the command-line](https://llm.datasette.io/en/stable/usage.html#usage-executing-prompts) - [Store prompts and responses in SQLite](https://llm.datasette.io/en/stable/logging.html#logging) - [Generate and store embeddings](https://llm.datasette.io/en/stable/embeddings/index.html#embeddings) - [Extract structured content from text and images](https://llm.datasette.io/en/stable/schemas.html#schemas) - [Grant models the ability to execute tools](https://llm.datasette.io/en/stable/tools.html#tools) - … and much, much more ## Quick start First, install LLM using `pip` or Homebrew or `pipx` or `uv`: ```bash pip install llm ``` Or with Homebrew (see [warning note](https://llm.datasette.io/en/stable/setup.html#homebrew-warning)): ```bash brew install llm ``` Or with [pipx](https://pypa.github.io/pipx/): ```bash pipx install llm ``` Or with [uv](https://docs.astral.sh/uv/guides/tools/) ```bash uv tool install llm ``` Use LLM to run prompts or start chats against an arbitrary OpenAI-compatible Chat Completions endpoint, such as [LM Studio](https://lmstudio.ai). With `uvx`, you can do this without installing LLM first: ```bash uvx llm openai endpoint http://localhost:1234/v1 \ -m google/gemma-4-12b \ "What is the capital of France?" uvx llm openai endpoint http://localhost:1234/v1 \ -m google/gemma-4-12b \ --chat ``` Add `--key your-api-key` if the endpoint requires authentication. See [Run against an endpoint without configuring it](https://llm.datasette.io/en/stable/other-models.html#openai-endpoint) for more options. If you have an [OpenAI API key](https://platform.openai.com/api-keys) key you can run this: ```bash # Paste your OpenAI API key into this llm keys set openai # Run a prompt (with the default gpt-5.6-luna model) llm "Ten fun names for a pet pelican" # Extract text from an image llm "extract text" -a scanned-document.jpg # Use a system prompt against a file cat myfile.py | llm -s "Explain this code" ``` Run prompts against [Gemini](https://aistudio.google.com/apikey) or [Anthropic](https://console.anthropic.com/) with their respective plugins: ```bash llm install llm-gemini llm keys set gemini # Paste Gemini API key here llm -m gemini-3.5-flash 'Tell me fun facts about Mountain View' llm install llm-anthropic llm keys set anthropic # Paste Anthropic API key here llm -m claude-sonnet-5 'Impress me with wild facts about turnips' ``` You can also [install a plugin](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#installing-plugins) to access models that can run on your local device. If you use [Ollama](https://ollama.com/): ```bash # Install the plugin llm install llm-ollama # Download and run a prompt against the Orca Mini 7B model ollama pull llama3.2:latest llm -m llama3.2:latest 'What is the capital of France?' ``` To start [an interactive chat](https://llm.datasette.io/en/stable/usage.html#usage-chat) with a model, use `llm chat`: ```bash llm chat -m gpt-4.1 ``` ```default Chatting with gpt-4.1 Type 'exit' or 'quit' to exit Type '!multi' to enter multiple lines, then '!end' to finish Type '!edit' to open your default editor and modify the prompt. Type '!fragment [ ...]' to insert one or more fragments > Tell me a joke about a pelican Why don't pelicans like to tip waiters? Because they always have a big bill! ``` ## Project news - 29th April 2026: [LLM 0.32a0 is a major backwards-compatible refactor](https://simonwillison.net/2026/Apr/29/llm/) - 11th August 2025: [LLM 0.27, the annotated release notes: GPT-5 and improved tool calling](https://simonwillison.net/2025/Aug/11/llm-027/) - 27th May 2025: [Large Language Models can run tools in your terminal with LLM 0.26](https://simonwillison.net/2025/May/27/llm-tools/) - 5th May 2025: [Feed a video to a vision LLM as a sequence of JPEG frames on the CLI (also LLM 0.25)](https://simonwillison.net/2025/May/5/llm-video-frames/) - 7th April 2025: [Long context support in LLM 0.24 using fragments and template plugins](https://simonwillison.net/2025/Apr/7/long-context-llm/) - 28th February 2025: [Structured data extraction from unstructured content using LLM schemas](https://simonwillison.net/2025/Feb/28/llm-schemas/) - 17th February 2025: [LLM 0.22, the annotated release notes](https://simonwillison.net/2025/Feb/17/llm/) - 29th October 2024: [You can now run prompts against images, audio and video in your terminal using LLM](https://simonwillison.net/2024/Oct/29/llm-multi-modal/) - 26th January 2024: [LLM 0.13: The annotated release notes](https://simonwillison.net/2024/Jan/26/llm/) - 12th September 2023: [Build an image search engine with llm-clip, chat with models with llm chat](https://simonwillison.net/2023/Sep/12/llm-clip-and-chat/) - 4th September 2023: [LLM now provides tools for working with embeddings](https://simonwillison.net/2023/Sep/4/llm-embeddings/) - 12th July 2023: [The LLM CLI tool now supports self-hosted language models via plugins](https://simonwillison.net/2023/Jul/12/llm/) - 18th May 2023: [llm, ttok and strip-tags—CLI tools for working with ChatGPT and other LLMs](https://simonwillison.net/2023/May/18/cli-tools-for-llms/) - 4th April 2023: [The original announcement of the llm CLI tool](https://simonwillison.net/2023/Apr/4/llm/) For everything else, see [the llm tag](https://simonwillison.net/tags/llm/) on my blog. ## Contents * [Setup](https://llm.datasette.io/en/stable/setup.html) * [Installation](https://llm.datasette.io/en/stable/setup.html#installation) * [Upgrading to the latest version](https://llm.datasette.io/en/stable/setup.html#upgrading-to-the-latest-version) * [Using uvx](https://llm.datasette.io/en/stable/setup.html#using-uvx) * [A note about Homebrew and PyTorch](https://llm.datasette.io/en/stable/setup.html#a-note-about-homebrew-and-pytorch) * [Installing plugins](https://llm.datasette.io/en/stable/setup.html#installing-plugins) * [API key management](https://llm.datasette.io/en/stable/setup.html#api-key-management) * [Saving and using stored keys](https://llm.datasette.io/en/stable/setup.html#saving-and-using-stored-keys) * [Passing keys using the –key option](https://llm.datasette.io/en/stable/setup.html#passing-keys-using-the-key-option) * [Keys in environment variables](https://llm.datasette.io/en/stable/setup.html#keys-in-environment-variables) * [Configuration](https://llm.datasette.io/en/stable/setup.html#configuration) * [Setting a custom default model](https://llm.datasette.io/en/stable/setup.html#setting-a-custom-default-model) * [Setting a custom directory location](https://llm.datasette.io/en/stable/setup.html#setting-a-custom-directory-location) * [Turning SQLite logging on and off](https://llm.datasette.io/en/stable/setup.html#turning-sqlite-logging-on-and-off) * [Usage](https://llm.datasette.io/en/stable/usage.html) * [Executing a prompt](https://llm.datasette.io/en/stable/usage.html#executing-a-prompt) * [Model options](https://llm.datasette.io/en/stable/usage.html#model-options) * [Attachments](https://llm.datasette.io/en/stable/usage.html#attachments) * [System prompts](https://llm.datasette.io/en/stable/usage.html#system-prompts) * [Tools](https://llm.datasette.io/en/stable/usage.html#tools) * [Extracting fenced code blocks](https://llm.datasette.io/en/stable/usage.html#extracting-fenced-code-blocks) * [JSON output](https://llm.datasette.io/en/stable/usage.html#json-output) * [Schemas](https://llm.datasette.io/en/stable/usage.html#schemas) * [Fragments](https://llm.datasette.io/en/stable/usage.html#fragments) * [Continuing a conversation](https://llm.datasette.io/en/stable/usage.html#continuing-a-conversation) * [Tips for using LLM with Bash or Zsh](https://llm.datasette.io/en/stable/usage.html#tips-for-using-llm-with-bash-or-zsh) * [Completion prompts](https://llm.datasette.io/en/stable/usage.html#completion-prompts) * [Starting an interactive chat](https://llm.datasette.io/en/stable/usage.html#starting-an-interactive-chat) * [Listing available models](https://llm.datasette.io/en/stable/usage.html#listing-available-models) * [Setting default options for models](https://llm.datasette.io/en/stable/usage.html#setting-default-options-for-models) * [OpenAI models](https://llm.datasette.io/en/stable/openai-models.html) * [Configuration](https://llm.datasette.io/en/stable/openai-models.html#configuration) * [OpenAI language models](https://llm.datasette.io/en/stable/openai-models.html#openai-language-models) * [Model features](https://llm.datasette.io/en/stable/openai-models.html#model-features) * [Web Search](https://llm.datasette.io/en/stable/openai-models.html#web-search) * [Code Interpreter](https://llm.datasette.io/en/stable/openai-models.html#code-interpreter) * [Fast mode and service tiers](https://llm.datasette.io/en/stable/openai-models.html#fast-mode-and-service-tiers) * [OpenAI embedding models](https://llm.datasette.io/en/stable/openai-models.html#openai-embedding-models) * [OpenAI completion models](https://llm.datasette.io/en/stable/openai-models.html#openai-completion-models) * [Adding more OpenAI models](https://llm.datasette.io/en/stable/openai-models.html#adding-more-openai-models) * [Other models](https://llm.datasette.io/en/stable/other-models.html) * [Installing and using a local model](https://llm.datasette.io/en/stable/other-models.html#installing-and-using-a-local-model) * [OpenAI-compatible models](https://llm.datasette.io/en/stable/other-models.html#openai-compatible-models) * [Run against an endpoint without configuring it](https://llm.datasette.io/en/stable/other-models.html#run-against-an-endpoint-without-configuring-it) * [Configure an OpenAI-compatible model](https://llm.datasette.io/en/stable/other-models.html#configure-an-openai-compatible-model) * [Extra HTTP headers](https://llm.datasette.io/en/stable/other-models.html#extra-http-headers) * [Tools](https://llm.datasette.io/en/stable/tools.html) * [How tools work](https://llm.datasette.io/en/stable/tools.html#how-tools-work) * [Trying out tools](https://llm.datasette.io/en/stable/tools.html#trying-out-tools) * [LLM’s implementation of tools](https://llm.datasette.io/en/stable/tools.html#llm-s-implementation-of-tools) * [Default tools](https://llm.datasette.io/en/stable/tools.html#default-tools) * [Tips for implementing tools](https://llm.datasette.io/en/stable/tools.html#tips-for-implementing-tools) * [Schemas](https://llm.datasette.io/en/stable/schemas.html) * [Schemas tutorial](https://llm.datasette.io/en/stable/schemas.html#schemas-tutorial) * [Getting started with dogs](https://llm.datasette.io/en/stable/schemas.html#getting-started-with-dogs) * [Extracting people from a news articles](https://llm.datasette.io/en/stable/schemas.html#extracting-people-from-a-news-articles) * [Using JSON schemas](https://llm.datasette.io/en/stable/schemas.html#using-json-schemas) * [Ways to specify a schema](https://llm.datasette.io/en/stable/schemas.html#ways-to-specify-a-schema) * [Concise LLM schema syntax](https://llm.datasette.io/en/stable/schemas.html#concise-llm-schema-syntax) * [Saving reusable schemas in templates](https://llm.datasette.io/en/stable/schemas.html#saving-reusable-schemas-in-templates) * [Browsing logged JSON objects created using schemas](https://llm.datasette.io/en/stable/schemas.html#browsing-logged-json-objects-created-using-schemas) * [Templates](https://llm.datasette.io/en/stable/templates.html) * [Getting started with –save](https://llm.datasette.io/en/stable/templates.html#getting-started-with-save) * [Using a template](https://llm.datasette.io/en/stable/templates.html#using-a-template) * [Listing available templates](https://llm.datasette.io/en/stable/templates.html#listing-available-templates) * [Templates as YAML files](https://llm.datasette.io/en/stable/templates.html#templates-as-yaml-files) * [System prompts](https://llm.datasette.io/en/stable/templates.html#system-prompts) * [Fragments](https://llm.datasette.io/en/stable/templates.html#fragments) * [Options](https://llm.datasette.io/en/stable/templates.html#options) * [Tools](https://llm.datasette.io/en/stable/templates.html#tools) * [Schemas](https://llm.datasette.io/en/stable/templates.html#schemas) * [Additional template variables](https://llm.datasette.io/en/stable/templates.html#additional-template-variables) * [Specifying default parameters](https://llm.datasette.io/en/stable/templates.html#specifying-default-parameters) * [Configuring code extraction](https://llm.datasette.io/en/stable/templates.html#configuring-code-extraction) * [Setting a default model for a template](https://llm.datasette.io/en/stable/templates.html#setting-a-default-model-for-a-template) * [Template loaders from plugins](https://llm.datasette.io/en/stable/templates.html#template-loaders-from-plugins) * [Fragments](https://llm.datasette.io/en/stable/fragments.html) * [Using fragments in a prompt](https://llm.datasette.io/en/stable/fragments.html#using-fragments-in-a-prompt) * [Using fragments in chat](https://llm.datasette.io/en/stable/fragments.html#using-fragments-in-chat) * [Browsing fragments](https://llm.datasette.io/en/stable/fragments.html#browsing-fragments) * [Setting aliases for fragments](https://llm.datasette.io/en/stable/fragments.html#setting-aliases-for-fragments) * [Viewing fragments in your logs](https://llm.datasette.io/en/stable/fragments.html#viewing-fragments-in-your-logs) * [Using fragments from plugins](https://llm.datasette.io/en/stable/fragments.html#using-fragments-from-plugins) * [Listing available fragment prefixes](https://llm.datasette.io/en/stable/fragments.html#listing-available-fragment-prefixes) * [Model aliases](https://llm.datasette.io/en/stable/aliases.html) * [Listing aliases](https://llm.datasette.io/en/stable/aliases.html#listing-aliases) * [Adding a new alias](https://llm.datasette.io/en/stable/aliases.html#adding-a-new-alias) * [Removing an alias](https://llm.datasette.io/en/stable/aliases.html#removing-an-alias) * [Viewing the aliases file](https://llm.datasette.io/en/stable/aliases.html#viewing-the-aliases-file) * [Embeddings](https://llm.datasette.io/en/stable/embeddings/index.html) * [Embedding with the CLI](https://llm.datasette.io/en/stable/embeddings/cli.html) * [llm embed](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed) * [llm embed-multi](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed-multi) * [llm similar](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-similar) * [llm embed-models](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed-models) * [llm collections list](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-collections-list) * [llm collections delete](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-collections-delete) * [Using embeddings from Python](https://llm.datasette.io/en/stable/embeddings/python-api.html) * [Working with collections](https://llm.datasette.io/en/stable/embeddings/python-api.html#working-with-collections) * [Retrieving similar items](https://llm.datasette.io/en/stable/embeddings/python-api.html#retrieving-similar-items) * [SQL schema](https://llm.datasette.io/en/stable/embeddings/python-api.html#sql-schema) * [Writing plugins to add new embedding models](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html) * [`EmbeddingModel`](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html#llm.EmbeddingModel) * [Embedding binary content](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html#embedding-binary-content) * [Embedding storage format](https://llm.datasette.io/en/stable/embeddings/storage.html) * [Plugins](https://llm.datasette.io/en/stable/plugins/index.html) * [Installing plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html) * [Listing installed plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#listing-installed-plugins) * [Running with a subset of plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#running-with-a-subset-of-plugins) * [Plugin directory](https://llm.datasette.io/en/stable/plugins/directory.html) * [Local models](https://llm.datasette.io/en/stable/plugins/directory.html#local-models) * [Remote APIs](https://llm.datasette.io/en/stable/plugins/directory.html#remote-apis) * [Tools](https://llm.datasette.io/en/stable/plugins/directory.html#tools) * [Fragments and template loaders](https://llm.datasette.io/en/stable/plugins/directory.html#fragments-and-template-loaders) * [Embedding models](https://llm.datasette.io/en/stable/plugins/directory.html#embedding-models) * [Extra commands](https://llm.datasette.io/en/stable/plugins/directory.html#extra-commands) * [Just for fun](https://llm.datasette.io/en/stable/plugins/directory.html#just-for-fun) * [Plugin hooks](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html) * [register_commands(cli)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-commands-cli) * [register_models(register, model_aliases)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-models-register-model-aliases) * [register_embedding_models(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-embedding-models-register) * [register_tools(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-tools-register) * [register_template_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-template-loaders-register) * [register_fragment_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-fragment-loaders-register) * [Developing a model plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html) * [The initial structure of the plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#the-initial-structure-of-the-plugin) * [Installing your plugin to try it out](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#installing-your-plugin-to-try-it-out) * [Building the Markov chain](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#building-the-markov-chain) * [Executing the Markov chain](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#executing-the-markov-chain) * [Adding that to the plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-that-to-the-plugin) * [Understanding execute()](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#understanding-execute) * [Prompts and responses are logged to the database](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#prompts-and-responses-are-logged-to-the-database) * [Adding options](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-options) * [Distributing your plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#distributing-your-plugin) * [GitHub repositories](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#github-repositories) * [Publishing plugins to PyPI](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#publishing-plugins-to-pypi) * [Adding metadata](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-metadata) * [What to do if it breaks](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#what-to-do-if-it-breaks) * [Advanced model plugins](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html) * [Tip: lazily load expensive dependencies](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tip-lazily-load-expensive-dependencies) * [Models that accept API keys](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#models-that-accept-api-keys) * [Async models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#async-models) * [Supporting schemas](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-schemas) * [Supporting tools](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-tools) * [Supporting server-side tools](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-server-side-tools) * [Attachments for multi-modal models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#attachments-for-multi-modal-models) * [Structured messages and streaming events](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#structured-messages-and-streaming-events) * [Condensing logged payloads with json_replacements](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#condensing-logged-payloads-with-json-replacements) * [Consuming prompt.messages in build_messages](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#consuming-prompt-messages-in-build-messages) * [Restoring opaque metadata on subsequent requests](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#restoring-opaque-metadata-on-subsequent-requests) * [Tracking token usage](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-token-usage) * [Tracking resolved model names](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-resolved-model-names) * [LLM_RAISE_ERRORS](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#llm-raise-errors) * [Utility functions for plugins](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html) * [llm.get_key()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-get-key) * [llm.user_dir()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-user-dir) * [llm.ModelError](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-modelerror) * [Response.fake()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#response-fake) * [Python API](https://llm.datasette.io/en/stable/python-api.html) * [Basic prompt execution](https://llm.datasette.io/en/stable/python-api.html#basic-prompt-execution) * [System prompts](https://llm.datasette.io/en/stable/python-api.html#system-prompts) * [Attachments](https://llm.datasette.io/en/stable/python-api.html#attachments) * [Tools](https://llm.datasette.io/en/stable/python-api.html#tools) * [Schemas](https://llm.datasette.io/en/stable/python-api.html#schemas) * [Fragments](https://llm.datasette.io/en/stable/python-api.html#fragments) * [Model options](https://llm.datasette.io/en/stable/python-api.html#model-options) * [Passing an API key](https://llm.datasette.io/en/stable/python-api.html#passing-an-api-key) * [Models from plugins](https://llm.datasette.io/en/stable/python-api.html#models-from-plugins) * [Accessing the underlying JSON](https://llm.datasette.io/en/stable/python-api.html#accessing-the-underlying-json) * [Token usage](https://llm.datasette.io/en/stable/python-api.html#token-usage) * [Streaming responses](https://llm.datasette.io/en/stable/python-api.html#streaming-responses) * [Structured messages and streaming events](https://llm.datasette.io/en/stable/python-api.html#structured-messages-and-streaming-events) * [Async models](https://llm.datasette.io/en/stable/python-api.html#async-models) * [`AsyncResponse`](https://llm.datasette.io/en/stable/python-api.html#llm.AsyncResponse) * [Tool functions can be sync or async](https://llm.datasette.io/en/stable/python-api.html#tool-functions-can-be-sync-or-async) * [Tool use for async models](https://llm.datasette.io/en/stable/python-api.html#tool-use-for-async-models) * [Conversations](https://llm.datasette.io/en/stable/python-api.html#conversations) * [Conversations using tools](https://llm.datasette.io/en/stable/python-api.html#conversations-using-tools) * [Listing models](https://llm.datasette.io/en/stable/python-api.html#listing-models) * [Running code when a response has completed](https://llm.datasette.io/en/stable/python-api.html#running-code-when-a-response-has-completed) * [Other functions](https://llm.datasette.io/en/stable/python-api.html#other-functions) * [set_alias(alias, model_id)](https://llm.datasette.io/en/stable/python-api.html#set-alias-alias-model-id) * [remove_alias(alias)](https://llm.datasette.io/en/stable/python-api.html#remove-alias-alias) * [set_default_model(alias)](https://llm.datasette.io/en/stable/python-api.html#set-default-model-alias) * [get_default_model()](https://llm.datasette.io/en/stable/python-api.html#get-default-model) * [set_default_embedding_model(alias) and get_default_embedding_model()](https://llm.datasette.io/en/stable/python-api.html#set-default-embedding-model-alias-and-get-default-embedding-model) * [Logging to SQLite](https://llm.datasette.io/en/stable/logging.html) * [Viewing the logs](https://llm.datasette.io/en/stable/logging.html#viewing-the-logs) * [-s/–short mode](https://llm.datasette.io/en/stable/logging.html#s-short-mode) * [Logs for a conversation](https://llm.datasette.io/en/stable/logging.html#logs-for-a-conversation) * [Searching the logs](https://llm.datasette.io/en/stable/logging.html#searching-the-logs) * [Filtering past a specific ID](https://llm.datasette.io/en/stable/logging.html#filtering-past-a-specific-id) * [Filtering by model](https://llm.datasette.io/en/stable/logging.html#filtering-by-model) * [Filtering by prompts that used specific fragments](https://llm.datasette.io/en/stable/logging.html#filtering-by-prompts-that-used-specific-fragments) * [Filtering by prompts that used specific tools](https://llm.datasette.io/en/stable/logging.html#filtering-by-prompts-that-used-specific-tools) * [Browsing data collected using schemas](https://llm.datasette.io/en/stable/logging.html#browsing-data-collected-using-schemas) * [Browsing logs using Datasette](https://llm.datasette.io/en/stable/logging.html#browsing-logs-using-datasette) * [Backing up your database](https://llm.datasette.io/en/stable/logging.html#backing-up-your-database) * [The message store](https://llm.datasette.io/en/stable/logging.html#the-message-store) * [Threads, turns, messages and parts](https://llm.datasette.io/en/stable/logging.html#threads-turns-messages-and-parts) * [A worked example](https://llm.datasette.io/en/stable/logging.html#a-worked-example) * [Content addressing as a contract](https://llm.datasette.io/en/stable/logging.html#content-addressing-as-a-contract) * [Forking and shared history](https://llm.datasette.io/en/stable/logging.html#forking-and-shared-history) * [Storage by reference](https://llm.datasette.io/en/stable/logging.html#storage-by-reference) * [The raw provider payload](https://llm.datasette.io/en/stable/logging.html#the-raw-provider-payload) * [Table by table](https://llm.datasette.io/en/stable/logging.html#table-by-table) * [Querying the message store](https://llm.datasette.io/en/stable/logging.html#querying-the-message-store) * [Logging from Python](https://llm.datasette.io/en/stable/logging.html#logging-from-python) * [SQL schema](https://llm.datasette.io/en/stable/logging.html#sql-schema) * [Related tools](https://llm.datasette.io/en/stable/related-tools.html) * [strip-tags](https://llm.datasette.io/en/stable/related-tools.html#strip-tags) * [ttok](https://llm.datasette.io/en/stable/related-tools.html#ttok) * [Symbex](https://llm.datasette.io/en/stable/related-tools.html#symbex) * [CLI reference](https://llm.datasette.io/en/stable/help.html) * [llm –help](https://llm.datasette.io/en/stable/help.html#llm-help) * [llm prompt –help](https://llm.datasette.io/en/stable/help.html#llm-prompt-help) * [llm chat –help](https://llm.datasette.io/en/stable/help.html#llm-chat-help) * [llm keys –help](https://llm.datasette.io/en/stable/help.html#llm-keys-help) * [llm logs –help](https://llm.datasette.io/en/stable/help.html#llm-logs-help) * [llm models –help](https://llm.datasette.io/en/stable/help.html#llm-models-help) * [llm templates –help](https://llm.datasette.io/en/stable/help.html#llm-templates-help) * [llm schemas –help](https://llm.datasette.io/en/stable/help.html#llm-schemas-help) * [llm tools –help](https://llm.datasette.io/en/stable/help.html#llm-tools-help) * [llm aliases –help](https://llm.datasette.io/en/stable/help.html#llm-aliases-help) * [llm fragments –help](https://llm.datasette.io/en/stable/help.html#llm-fragments-help) * [llm plugins –help](https://llm.datasette.io/en/stable/help.html#llm-plugins-help) * [llm install –help](https://llm.datasette.io/en/stable/help.html#llm-install-help) * [llm uninstall –help](https://llm.datasette.io/en/stable/help.html#llm-uninstall-help) * [llm embed –help](https://llm.datasette.io/en/stable/help.html#llm-embed-help) * [llm embed-multi –help](https://llm.datasette.io/en/stable/help.html#llm-embed-multi-help) * [llm similar –help](https://llm.datasette.io/en/stable/help.html#llm-similar-help) * [llm embed-models –help](https://llm.datasette.io/en/stable/help.html#llm-embed-models-help) * [llm collections –help](https://llm.datasette.io/en/stable/help.html#llm-collections-help) * [llm openai –help](https://llm.datasette.io/en/stable/help.html#llm-openai-help) * [Contributing](https://llm.datasette.io/en/stable/contributing.html) * [Updating recorded HTTP API interactions and associated snapshots](https://llm.datasette.io/en/stable/contributing.html#updating-recorded-http-api-interactions-and-associated-snapshots) * [Debugging tricks](https://llm.datasette.io/en/stable/contributing.html#debugging-tricks) * [Documentation](https://llm.datasette.io/en/stable/contributing.html#documentation) * [Release process](https://llm.datasette.io/en/stable/contributing.html#release-process) * [Changelog](https://llm.datasette.io/en/stable/changelog.html) ././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1785864218.582707 llm-0.32/llm/0000755000175100017510000000000015234420033012450 5ustar00runnerrunner././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/__init__.py0000644000175100017510000003521315234420023014564 0ustar00runnerrunnerimport inspect import json import os import pathlib import struct from collections.abc import Callable from typing import Any import click from .embeddings import Collection from .errors import ( ModelError, NeedsKeyException, ) from .hookspecs import hookimpl from .models import ( AsyncConversation, AsyncKeyModel, AsyncModel, AsyncResponse, Attachment, CancelToolCall, Conversation, EmbeddingModel, EmbeddingModelWithAliases, KeyModel, Model, ModelWithAliases, Options, PauseChain, Prompt, Response, ServerSideTool, Tool, Toolbox, ToolCall, ToolOutput, ToolResult, Usage, ) from .parts import ( Message, assistant, system, tool_message, user, ) from .plugins import load_plugins, pm from .templates import Template from .utils import Fragment, schema_dsl __all__ = [ "AsyncConversation", "AsyncKeyModel", "AsyncModel", "AsyncResponse", "Attachment", "CancelToolCall", "Collection", "Conversation", "Fragment", "KeyModel", "Message", "Model", "ModelError", "NeedsKeyException", "Options", "PauseChain", "Prompt", "Response", "ServerSideTool", "Template", "Tool", "ToolCall", "ToolOutput", "ToolResult", "Toolbox", "Usage", "assistant", "get_async_model", "get_key", "get_model", "hookimpl", "schema_dsl", "system", "tool_message", "user", "user_dir", ] DEFAULT_MODEL = "gpt-5.6-luna" def get_plugins(all=False): plugins = [] plugin_to_distinfo = dict(pm.list_plugin_distinfo()) for plugin in pm.get_plugins(): if not all and plugin.__name__.startswith("llm.default_plugins."): continue plugin_info = { "name": plugin.__name__, "hooks": [h.name for h in pm.get_hookcallers(plugin)], } distinfo = plugin_to_distinfo.get(plugin) if distinfo: plugin_info["version"] = distinfo.version plugin_info["name"] = ( getattr(distinfo, "name", None) or distinfo.project_name ) plugins.append(plugin_info) return plugins def get_models_with_aliases() -> list["ModelWithAliases"]: model_aliases = [] # Include aliases from aliases.json aliases_path = user_dir() / "aliases.json" extra_model_aliases: dict[str, list] = {} if aliases_path.exists(): configured_aliases = json.loads(aliases_path.read_text()) for alias, model_id in configured_aliases.items(): extra_model_aliases.setdefault(model_id, []).append(alias) def register(model, async_model=None, aliases=None): alias_list = list(aliases or []) if model.model_id in extra_model_aliases: alias_list.extend(extra_model_aliases[model.model_id]) model_aliases.append(ModelWithAliases(model, async_model, alias_list)) load_plugins() pm.hook.register_models(register=register, model_aliases=model_aliases) return model_aliases def _get_loaders(hook_method) -> dict[str, Callable]: load_plugins() loaders = {} def register(prefix, loader): suffix = 0 prefix_to_try = prefix while prefix_to_try in loaders: suffix += 1 prefix_to_try = f"{prefix}_{suffix}" loaders[prefix_to_try] = loader hook_method(register=register) return loaders def get_template_loaders() -> dict[str, Callable[[str], Template]]: """Get template loaders registered by plugins.""" return _get_loaders(pm.hook.register_template_loaders) def get_fragment_loaders() -> dict[ str, Callable[[str], Fragment | Attachment | list[Fragment | Attachment]], ]: """Get fragment loaders registered by plugins.""" return _get_loaders(pm.hook.register_fragment_loaders) def get_tools() -> dict[str, Tool | type[Toolbox]]: """Return all tools (llm.Tool and llm.Toolbox) registered by plugins.""" load_plugins() tools: dict[str, Tool | type[Toolbox]] = {} # Variable to track current plugin name current_plugin_name = None def register( tool_or_function: Tool | type[Toolbox] | Callable[..., Any], name: str | None = None, ) -> None: tool: Tool | type[Toolbox] | None = None # If it's a Toolbox class, set the plugin field on it if inspect.isclass(tool_or_function): if issubclass(tool_or_function, Toolbox): tool = tool_or_function if current_plugin_name: tool.plugin = current_plugin_name tool.name = name or tool.__name__ else: raise TypeError( f"Toolbox classes must inherit from llm.Toolbox, {tool_or_function.__name__} does not." ) # If it's already a Tool instance, use it directly elif isinstance(tool_or_function, Tool): tool = tool_or_function if name: tool.name = name if current_plugin_name: tool.plugin = current_plugin_name # If it's a bare function, wrap it in a Tool else: tool = Tool.function(tool_or_function, name=name) if current_plugin_name: tool.plugin = current_plugin_name # Get the name for the tool/toolbox if tool: # For Toolbox classes, use their name attribute or class name if inspect.isclass(tool) and issubclass(tool, Toolbox): prefix = name or getattr(tool, "name", tool.__name__) or "" else: prefix = name or tool.name or "" suffix = 0 candidate = prefix # Avoid name collisions while candidate in tools: suffix += 1 candidate = f"{prefix}_{suffix}" tools[candidate] = tool # Call each plugin's register_tools hook individually to track current_plugin_name for plugin in pm.get_plugins(): current_plugin_name = pm.get_name(plugin) hook_caller = pm.hook.register_tools plugin_impls = [ impl for impl in hook_caller.get_hookimpls() if impl.plugin is plugin ] for impl in plugin_impls: impl.function(register=register) return tools def get_embedding_models_with_aliases() -> list["EmbeddingModelWithAliases"]: model_aliases = [] # Include aliases from aliases.json aliases_path = user_dir() / "aliases.json" extra_model_aliases: dict[str, list] = {} if aliases_path.exists(): configured_aliases = json.loads(aliases_path.read_text()) for alias, model_id in configured_aliases.items(): extra_model_aliases.setdefault(model_id, []).append(alias) def register(model, aliases=None): alias_list = list(aliases or []) if model.model_id in extra_model_aliases: alias_list.extend(extra_model_aliases[model.model_id]) model_aliases.append(EmbeddingModelWithAliases(model, alias_list)) load_plugins() pm.hook.register_embedding_models(register=register) return model_aliases def get_embedding_models(): models = [] def register(model, aliases=None): models.append(model) load_plugins() pm.hook.register_embedding_models(register=register) return models def get_embedding_model(name): aliases = get_embedding_model_aliases() try: return aliases[name] except KeyError: raise UnknownModelError("Unknown model: " + str(name)) def get_embedding_model_aliases() -> dict[str, EmbeddingModel]: model_aliases = {} for model_with_aliases in get_embedding_models_with_aliases(): for alias in model_with_aliases.aliases: model_aliases[alias] = model_with_aliases.model model_aliases[model_with_aliases.model.model_id] = model_with_aliases.model return model_aliases def get_async_model_aliases() -> dict[str, AsyncModel]: async_model_aliases = {} for model_with_aliases in get_models_with_aliases(): if model_with_aliases.async_model: for alias in model_with_aliases.aliases: async_model_aliases[alias] = model_with_aliases.async_model async_model_aliases[model_with_aliases.model.model_id] = ( model_with_aliases.async_model ) return async_model_aliases def get_model_aliases() -> dict[str, Model]: model_aliases = {} for model_with_aliases in get_models_with_aliases(): if model_with_aliases.model: for alias in model_with_aliases.aliases: model_aliases[alias] = model_with_aliases.model model_aliases[model_with_aliases.model.model_id] = model_with_aliases.model return model_aliases class UnknownModelError(KeyError): pass def get_models() -> list[Model]: "Get all registered models" models_with_aliases = get_models_with_aliases() return [mwa.model for mwa in models_with_aliases if mwa.model] def get_async_models() -> list[AsyncModel]: "Get all registered async models" models_with_aliases = get_models_with_aliases() return [mwa.async_model for mwa in models_with_aliases if mwa.async_model] def get_async_model(name: str | None = None) -> AsyncModel: "Get an async model by name or alias" aliases = get_async_model_aliases() name = name or get_default_model() try: return aliases[name] except KeyError: # Does a sync model exist? sync_model = None try: sync_model = get_model(name, _skip_async=True) except UnknownModelError: pass if sync_model: raise UnknownModelError("Unknown async model (sync model exists): " + name) else: raise UnknownModelError("Unknown model: " + name) def get_model(name: str | None = None, _skip_async: bool = False) -> Model: "Get a model by name or alias" aliases = get_model_aliases() name = name or get_default_model() try: return aliases[name] except KeyError: # Does an async model exist? if _skip_async: raise UnknownModelError("Unknown model: " + name) async_model = None try: async_model = get_async_model(name) except UnknownModelError: pass if async_model: raise UnknownModelError("Unknown model (async model exists): " + name) else: raise UnknownModelError("Unknown model: " + name) def get_key( explicit_key: str | None = None, key_alias: str | None = None, env_var: str | None = None, *, alias: str | None = None, env: str | None = None, input: str | None = None, ) -> str | None: """ Return an API key based on a hierarchy of potential sources. You should use the keyword arguments, the positional arguments are here purely for backwards-compatibility with older code. :param input: Input provided by the user. This may be the key, or an alias of a key in keys.json. :param alias: The alias used to retrieve the key from the keys.json file. :param env: Name of the environment variable to check for the key as a final fallback. """ if alias: key_alias = alias if env: env_var = env if input: explicit_key = input stored_keys = load_keys() # If user specified an alias, use the key stored for that alias if explicit_key in stored_keys: return stored_keys[explicit_key] if explicit_key: # User specified a key that's not an alias, use that return explicit_key # Stored key over-rides environment variables over-ride the default key if key_alias in stored_keys: return stored_keys[key_alias] # Finally try environment variable if env_var and os.environ.get(env_var): return os.environ[env_var] # Couldn't find it return None def load_keys(): path = user_dir() / "keys.json" if path.exists(): return json.loads(path.read_text()) else: return {} def user_dir(): llm_user_path = os.environ.get("LLM_USER_PATH") if llm_user_path: path = pathlib.Path(llm_user_path) else: path = pathlib.Path(click.get_app_dir("io.datasette.llm")) path.mkdir(exist_ok=True, parents=True) return path def set_alias(alias, model_id_or_alias): """ Set an alias to point to the specified model. """ path = user_dir() / "aliases.json" path.parent.mkdir(parents=True, exist_ok=True) if not path.exists(): path.write_text("{}\n") try: current = json.loads(path.read_text()) except json.decoder.JSONDecodeError: # We're going to write a valid JSON file in a moment: current = {} # Resolve model_id_or_alias to a model_id try: model = get_model(model_id_or_alias) model_id = model.model_id except UnknownModelError: # Try to resolve it to an embedding model try: model = get_embedding_model(model_id_or_alias) model_id = model.model_id except UnknownModelError: # Set the alias to the exact string they provided instead model_id = model_id_or_alias current[alias] = model_id path.write_text(json.dumps(current, indent=4) + "\n") def remove_alias(alias): """ Remove an alias. """ path = user_dir() / "aliases.json" if not path.exists(): raise KeyError("No aliases.json file exists") try: current = json.loads(path.read_text()) except json.decoder.JSONDecodeError: raise KeyError("aliases.json file is not valid JSON") if alias not in current: raise KeyError(f"No such alias: {alias}") del current[alias] path.write_text(json.dumps(current, indent=4) + "\n") def encode(values): return struct.pack("<" + "f" * len(values), *values) def decode(binary): return struct.unpack("<" + "f" * (len(binary) // 4), binary) def cosine_similarity(a, b): dot_product = sum(x * y for x, y in zip(a, b)) magnitude_a = sum(x * x for x in a) ** 0.5 magnitude_b = sum(x * x for x in b) ** 0.5 return dot_product / (magnitude_a * magnitude_b) def get_default_model(filename="default_model.txt", default=DEFAULT_MODEL): path = user_dir() / filename if path.exists(): return path.read_text().strip() else: return default def set_default_model(model, filename="default_model.txt"): path = user_dir() / filename if model is None and path.exists(): path.unlink() else: path.write_text(model) def get_default_embedding_model(): return get_default_model("default_embedding_model.txt", None) def set_default_embedding_model(model): set_default_model(model, "default_embedding_model.txt") ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/__main__.py0000644000175100017510000000007315234420023014541 0ustar00runnerrunnerfrom .cli import cli if __name__ == "__main__": cli() ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/cli.py0000644000175100017510000042023415234420023013575 0ustar00runnerrunnerimport asyncio import base64 import inspect import io import json import os import pathlib import re import readline import shutil import sqlite3 import sys import textwrap import warnings from collections.abc import Iterable from dataclasses import asdict from importlib.metadata import version from runpy import run_module from typing import Any, cast import click import httpx import pydantic import sqlite_utils import yaml from click_default_group import DefaultGroup from sqlite_utils.utils import Format, rows_from_file from llm import ( AsyncConversation, AsyncKeyModel, AsyncResponse, Attachment, CancelToolCall, Collection, Conversation, Fragment, KeyModel, Response, ServerSideTool, Template, Tool, Toolbox, UnknownModelError, encode, get_async_model, get_default_embedding_model, get_default_model, get_embedding_model, get_embedding_model_aliases, get_embedding_models_with_aliases, get_fragment_loaders, get_model, get_model_aliases, get_models_with_aliases, get_plugins, get_template_loaders, get_tools, remove_alias, set_alias, set_default_embedding_model, set_default_model, user_dir, ) from llm.models import ChainResponse, _BaseChainResponse, _BaseConversation from .logs import ( LogStore, legacy_log_row_extras, log_row_extras, merged_log_rows, ) from .migrations import migrate from .plugins import load_plugins, pm from .utils import ( ensure_fragment, extract_fenced_code_block, find_unused_key, has_plugin_prefix, instantiate_from_spec, make_schema_id, maybe_fenced_code, mimetype_from_path, mimetype_from_string, multi_schema, output_rows_as_json, resolve_schema_input, schema_dsl, schema_summary, token_usage_string, truncate_string, ) warnings.simplefilter("ignore", ResourceWarning) DEFAULT_TEMPLATE = "prompt: " class FragmentNotFound(Exception): pass def display_stream_events(events, *, show_reasoning=True): """Consume a sync iterator of StreamEvents and write them. Text events go to stdout. Reasoning events go to stderr in dim style. A newline is written to stderr at each reasoning→text transition so the assistant text starts on a fresh visual line. """ was_reasoning = False for event in events: if event.type == "text": if was_reasoning and show_reasoning: click.echo("", err=True) was_reasoning = False click.echo(event.chunk, nl=False) elif event.type == "reasoning" and show_reasoning: was_reasoning = True click.echo(click.style(event.chunk, dim=True), nl=False, err=True) async def display_async_stream_events(events, *, show_reasoning=True): """Async counterpart of display_stream_events.""" was_reasoning = False async for event in events: if event.type == "text": if was_reasoning and show_reasoning: click.echo("", err=True) was_reasoning = False click.echo(event.chunk, nl=False) elif event.type == "reasoning" and show_reasoning: was_reasoning = True click.echo(click.style(event.chunk, dim=True), nl=False, err=True) def _run_chat( model_label, prompt_callback, *, db=None, initial_fragments=None, initial_attachments=None, transform_prompt=None, after_response=None, show_reasoning=True, ): """Run the terminal chat loop shared by managed and transient models.""" click.echo(f"Chatting with {model_label}") click.echo("Type 'exit' or 'quit' to exit") click.echo("Type '!multi' to enter multiple lines, then '!end' to finish") click.echo("Type '!edit' to open your default editor and modify the prompt") if db is not None: click.echo( "Type '!fragment [ ...]' to insert one or more fragments" ) argument_fragments = list(initial_fragments or []) argument_attachments = list(initial_attachments or []) in_multi = False accumulated = [] accumulated_fragments = [] accumulated_attachments = [] end_token = "!end" while True: prompt = click.prompt("", prompt_suffix="> " if not in_multi else "") fragments = [] attachments = [] if argument_fragments: fragments += argument_fragments # Fragments from command options are added to the first message only. argument_fragments = [] if argument_attachments: attachments = argument_attachments argument_attachments = [] if prompt.strip().startswith("!multi"): in_multi = True bits = prompt.strip().split() if len(bits) > 1: end_token = "!end {}".format(" ".join(bits[1:])) continue if prompt.strip() == "!edit": edited_prompt = click.edit() if edited_prompt is None: click.echo("Editor closed without saving.", err=True) continue prompt = edited_prompt.strip() if db is not None and prompt.strip().startswith("!fragment "): prompt, fragments, attachments = process_fragments_in_chat(db, prompt) if in_multi: if prompt.strip() == end_token: prompt = "\n".join(accumulated) fragments = accumulated_fragments attachments = accumulated_attachments in_multi = False accumulated = [] accumulated_fragments = [] accumulated_attachments = [] else: if prompt: accumulated.append(prompt) accumulated_fragments += fragments accumulated_attachments += attachments continue if prompt.strip() in ("exit", "quit"): break if transform_prompt is not None: prompt = transform_prompt(prompt) response = prompt_callback(prompt, fragments, attachments) display_stream_events( response.stream_events(), show_reasoning=show_reasoning, ) if after_response is not None: after_response(response) print() def validate_fragment_alias(ctx, param, value): if not re.match(r"^[a-zA-Z0-9_-]+$", value): raise click.BadParameter("Fragment alias must be alphanumeric") return value def resolve_fragments( db: sqlite_utils.Database, fragments: Iterable[str], allow_attachments: bool = False ) -> list[Fragment | Attachment]: """ Resolve fragment strings into a mixed of llm.Fragment() and llm.Attachment() objects. """ def _load_by_alias(fragment: str) -> tuple[str | None, str | None]: rows = list( db.query( """ select content, source from fragments left join fragment_aliases on fragments.id = fragment_aliases.fragment_id where alias = :alias or hash = :alias limit 1 """, {"alias": fragment}, ) ) if rows: row = rows[0] return row["content"], row["source"] return None, None # The fragment strings could be URLs or paths or plugin references resolved: list[Fragment | Attachment] = [] for fragment in fragments: if fragment.startswith(("http://", "https://")): llm_version = version("llm") headers = {"User-Agent": f"llm/{llm_version} (https://llm.datasette.io/)"} client = httpx.Client( follow_redirects=True, max_redirects=3, headers=headers ) response = client.get(fragment) response.raise_for_status() resolved.append(Fragment(response.text, fragment)) elif fragment == "-": resolved.append(Fragment(sys.stdin.read(), "-")) elif has_plugin_prefix(fragment) and not pathlib.Path(fragment).exists(): prefix, rest = fragment.split(":", 1) loaders = get_fragment_loaders() if prefix not in loaders: raise FragmentNotFound(f"Unknown fragment prefix: {prefix}") loader = loaders[prefix] try: result = loader(rest) if not isinstance(result, list): result = [result] if not allow_attachments and any( isinstance(r, Attachment) for r in result ): raise FragmentNotFound( f"Fragment loader {prefix} returned a disallowed attachment" ) resolved.extend(result) except Exception as ex: # noqa: BLE001 raise FragmentNotFound(f"Could not load fragment {fragment}: {ex}") else: # Try from the DB content, source = _load_by_alias(fragment) if content is not None: resolved.append(Fragment(content, source)) else: # Now try path path = pathlib.Path(fragment) if path.exists(): resolved.append(Fragment(path.read_text(), str(path.resolve()))) else: raise FragmentNotFound(f"Fragment '{fragment}' not found") return resolved def process_fragments_in_chat( db: sqlite_utils.Database, prompt: str ) -> tuple[str, list[Fragment], list[Attachment]]: """ Process any !fragment commands in a chat prompt and return the modified prompt plus resolved fragments and attachments. """ prompt_lines = [] fragments = [] attachments = [] for line in prompt.splitlines(): if line.startswith("!fragment "): try: fragment_strs = line.strip().removeprefix("!fragment ").split() fragments_and_attachments = resolve_fragments( db, fragments=fragment_strs, allow_attachments=True ) fragments += [ fragment for fragment in fragments_and_attachments if isinstance(fragment, Fragment) ] attachments += [ attachment for attachment in fragments_and_attachments if isinstance(attachment, Attachment) ] except FragmentNotFound as ex: raise click.ClickException(str(ex)) else: prompt_lines.append(line) return "\n".join(prompt_lines), fragments, attachments class AttachmentError(Exception): """Exception raised for errors in attachment resolution.""" def resolve_attachment(value): """ Resolve an attachment from a string value which could be: - "-" for stdin - A URL - A file path Returns an Attachment object. Raises AttachmentError if the attachment cannot be resolved. """ if value == "-": content = sys.stdin.buffer.read() # Try to guess type mimetype = mimetype_from_string(content) if mimetype is None: raise AttachmentError("Could not determine mimetype of stdin") return Attachment(type=mimetype, path=None, url=None, content=content) if "://" in value: # Confirm URL exists and try to guess type try: response = httpx.head(value) response.raise_for_status() mimetype = response.headers.get("content-type") except httpx.HTTPError as ex: raise AttachmentError(str(ex)) return Attachment(type=mimetype, path=None, url=value, content=None) # Check that the file exists path = pathlib.Path(value) if not path.exists(): raise AttachmentError(f"File {value} does not exist") path = path.resolve() # Try to guess type mimetype = mimetype_from_path(str(path)) if mimetype is None: raise AttachmentError(f"Could not determine mimetype of {value}") return Attachment(type=mimetype, path=str(path), url=None, content=None) class AttachmentType(click.ParamType): name = "attachment" def convert(self, value, param, ctx): try: return resolve_attachment(value) except AttachmentError as e: self.fail(str(e), param, ctx) def resolve_attachment_with_type(value: str, mimetype: str) -> Attachment: if "://" in value: attachment = Attachment(mimetype, None, value, None) elif value == "-": content = sys.stdin.buffer.read() attachment = Attachment(mimetype, None, None, content) else: # Look for file path = pathlib.Path(value) if not path.exists(): raise click.BadParameter(f"File {value} does not exist") path = path.resolve() attachment = Attachment(mimetype, str(path), None, None) return attachment def attachment_types_callback(ctx, param, values) -> list[Attachment]: collected = [] for value, mimetype in values: collected.append(resolve_attachment_with_type(value, mimetype)) return collected def _apply_template(template, prompt, params, system): """Apply a loaded template to a prompt and system prompt.""" try: uses_input = "input" in template.vars() input_ = prompt if uses_input else "" template_prompt, template_system = template.evaluate(input_, params) except Template.MissingVariables as ex: raise click.ClickException(str(ex)) if template_system and not system: system = template_system if template_prompt: if prompt and not uses_input: prompt = f"{template_prompt}\n{prompt}" else: prompt = template_prompt return prompt, system def _merge_template_options(template, options): """Add template options unless the same option was provided explicitly.""" merged_options = list(options) specified_options = dict(merged_options) for option_name, option_value in (template.options or {}).items(): if option_name not in specified_options: merged_options.append((option_name, option_value)) return merged_options def _merge_template_attachments(template, attachments, attachment_types): """Resolve and prepend attachments declared by a loaded template.""" if template.attachments: attachments = [ resolve_attachment(value) for value in template.attachments ] + list(attachments) if template.attachment_types: attachment_types = [ resolve_attachment_with_type(item.value, item.type) for item in template.attachment_types ] + list(attachment_types) return attachments, attachment_types def _merge_template_tools(template, tools, python_tools): """Prepend trusted tool definitions declared by a loaded template.""" if template.tools: tools = [*template.tools, *tools] if template.functions and template._functions_is_trusted: python_tools = [template.functions, *python_tools] return tools, python_tools def json_validator(object_name): def validator(ctx, param, value): if value is None: return value try: obj = json.loads(value) if not isinstance(obj, dict): raise click.BadParameter(f"{object_name} must be a JSON object") return obj except json.JSONDecodeError: raise click.BadParameter(f"{object_name} must be valid JSON") return validator def schema_option(fn): click.option( "schema_input", "--schema", help="JSON schema, filepath or ID", )(fn) return fn def tool_options(fn): """Add the shared CLI options for selecting and executing tools.""" decorators = ( click.option( "tools", "-T", "--tool", multiple=True, help="Name of a tool to make available to the model", ), click.option( "python_tools", "--functions", multiple=True, help="Python code block or file path defining functions to register as tools", ), click.option( "tools_debug", "--td", "--tools-debug", is_flag=True, help="Show full details of tool executions", envvar="LLM_TOOLS_DEBUG", ), click.option( "tools_approve", "--ta", "--tools-approve", is_flag=True, help="Manually approve every tool execution", ), click.option( "chain_limit", "--cl", "--chain-limit", type=int, default=5, help=( "How many chained tool responses to allow, " "default 5, set 0 for unlimited" ), ), ) for decorator in reversed(decorators): fn = decorator(fn) return fn @click.group( cls=DefaultGroup, default="prompt", default_if_no_args=True, context_settings={"help_option_names": ["-h", "--help"]}, ) @click.version_option() def cli(): """ Access Large Language Models from the command-line Documentation: https://llm.datasette.io/ LLM can run models from many different providers. Consult the plugin directory for a list of available models: https://llm.datasette.io/en/stable/plugins/directory.html To get started with OpenAI, obtain an API key from them and: \b $ llm keys set openai Enter key: ... Then execute a prompt like this: llm 'Five outrageous names for a pet pelican' For a full list of prompting options run: llm prompt --help """ @cli.command(name="prompt") @click.argument("prompt", required=False) @click.option("-s", "--system", help="System prompt to use") @click.option("model_id", "-m", "--model", help="Model to use", envvar="LLM_MODEL") @click.option( "-d", "--database", type=click.Path(readable=True, dir_okay=False), help="Path to log database", ) @click.option( "queries", "-q", "--query", multiple=True, help="Use first model matching these strings", ) @click.option( "attachments", "-a", "--attachment", type=AttachmentType(), multiple=True, help="Attachment path or URL or -", ) @click.option( "attachment_types", "--at", "--attachment-type", type=(str, str), multiple=True, callback=attachment_types_callback, help="\b\nAttachment with explicit mimetype,\n--at image.jpg image/jpeg", ) @tool_options @click.option( "options", "-o", "--option", type=(str, str), multiple=True, help="key/value options for the model", ) @click.option( "show_model_options", "--options", is_flag=True, help="Show options for the selected model", ) @schema_option @click.option( "--schema-multi", help="JSON schema to use for multiple results", ) @click.option( "fragments", "-f", "--fragment", multiple=True, help="Fragment (alias, URL, hash or file path) to add to the prompt", ) @click.option( "system_fragments", "--sf", "--system-fragment", multiple=True, help="Fragment to add to system prompt", ) @click.option("-t", "--template", help="Template to use") @click.option( "-p", "--param", multiple=True, type=(str, str), help="Parameters for template", ) @click.option("--no-stream", is_flag=True, help="Do not stream output") @click.option("-n", "--no-log", is_flag=True, help="Don't log to database") @click.option("--log", is_flag=True, help="Log prompt and response to the database") @click.option("-R", "--hide-reasoning", is_flag=True, help="Hide reasoning output") @click.option( "_continue", "-c", "--continue", is_flag=True, flag_value=-1, help="Continue the most recent conversation.", ) @click.option( "conversation_id", "--cid", "--conversation", help="Continue the conversation with the given ID.", ) @click.option("--key", help="API key to use") @click.option("--save", help="Save prompt with this template name") @click.option("async_", "--async", is_flag=True, help="Run prompt asynchronously") @click.option("-u", "--usage", is_flag=True, help="Show token usage") @click.option("-x", "--extract", is_flag=True, help="Extract first fenced code block") @click.option( "extract_last", "--xl", "--extract-last", is_flag=True, help="Extract last fenced code block", ) @click.option( "json_output", "--json", is_flag=True, help="Output the response as JSON, same format as llm logs --json", ) def prompt( prompt, system, model_id, database, queries, attachments, attachment_types, tools, python_tools, tools_debug, tools_approve, chain_limit, options, show_model_options, schema_input, schema_multi, fragments, system_fragments, template, param, no_stream, no_log, log, hide_reasoning, _continue, conversation_id, key, save, async_, usage, extract, extract_last, json_output, ): """ Execute a prompt Documentation: https://llm.datasette.io/en/stable/usage.html Examples: \b llm 'Capital of France?' llm 'Capital of France?' -m gpt-5.5 llm 'Capital of France?' -s 'answer in Spanish' Multi-modal models can be called with attachments like this: \b llm 'Extract text from this image' -a image.jpg llm 'Describe' -a https://static.simonwillison.net/static/2024/pelicans.jpg cat image | llm 'describe image' -a - # With an explicit mimetype: cat image | llm 'describe image' --at - image/jpeg The -x/--extract option returns just the content of the first ``` fenced code block, if one is present. If none are present it returns the full response. \b llm 'JavaScript function for reversing a string' -x """ if log and no_log: raise click.ClickException("--log and --no-log are mutually exclusive") if queries and not model_id: # Use -q options to find model with shortest model_id matches = [] for model_with_aliases in get_models_with_aliases(): if all(model_with_aliases.matches(q) for q in queries): matches.append(model_with_aliases.model.model_id) if not matches: raise click.ClickException( "No model found matching queries {}".format(", ".join(queries)) ) model_id = min(matches, key=len) if show_model_options and not (conversation_id or _continue or template): model_id = model_id or get_default_model() try: if async_: get_async_model(model_id) else: get_model(model_id) except UnknownModelError as ex: raise click.ClickException(ex) click.echo(render_model_with_options(model_id, async_=async_)) return log_path = pathlib.Path(database) if database else logs_db_path() (log_path.parent).mkdir(parents=True, exist_ok=True) db = sqlite_utils.Database(log_path) migrate(db) if schema_multi: schema_input = schema_multi schema = resolve_schema_input(db, schema_input, load_template) if schema_multi: # Convert that schema into multiple "items" of the same schema schema = multi_schema(schema) def read_prompt(): nonlocal prompt, schema # Is there extra prompt available on stdin? stdin_prompt = None if not sys.stdin.isatty(): stdin_prompt = sys.stdin.read() if stdin_prompt: bits = [stdin_prompt] if prompt: bits.append(prompt) prompt = " ".join(bits) if ( prompt is None and not save and sys.stdin.isatty() and not attachments and not attachment_types and not schema and not fragments ): # Hang waiting for input to stdin (unless --save) prompt = sys.stdin.read() return prompt if save: # We are saving their prompt/system/etc to a new template # Fields to save: prompt, system, model - and more in the future disallowed_options = [] for option, var in ( ("--template", template), ("--continue", _continue), ("--cid", conversation_id), ): if var: disallowed_options.append(option) if disallowed_options: raise click.ClickException( "--save cannot be used with {}".format(", ".join(disallowed_options)) ) path = template_dir() / f"{save}.yaml" to_save = {} if model_id: model_aliases = get_model_aliases() try: to_save["model"] = model_aliases[model_id].model_id except KeyError: raise click.ClickException(f"'{model_id}' is not a known model") prompt = read_prompt() if prompt: to_save["prompt"] = prompt if system: to_save["system"] = system if param: to_save["defaults"] = dict(param) if extract: to_save["extract"] = True if extract_last: to_save["extract_last"] = True if schema: to_save["schema_object"] = schema if fragments: to_save["fragments"] = list(fragments) if system_fragments: to_save["system_fragments"] = list(system_fragments) if python_tools: to_save["functions"] = "\n\n".join(python_tools) if tools: to_save["tools"] = list(tools) if attachments: # Only works for attachments with a path or url to_save["attachments"] = [ (a.path or a.url) for a in attachments if (a.path or a.url) ] if attachment_types: to_save["attachment_types"] = [ {"type": a.type, "value": a.path or a.url} for a in attachment_types if (a.path or a.url) ] if options: # Need to validate and convert their types first model = get_model(model_id or get_default_model()) try: options_model = model.Options(**dict(options)) # Use model_dump(mode="json") so Enums become their .value strings to_save["options"] = { k: v for k, v in options_model.model_dump(mode="json").items() if v is not None } except pydantic.ValidationError as ex: raise click.ClickException(render_errors(ex.errors())) path.write_text( yaml.safe_dump( to_save, indent=4, default_flow_style=False, sort_keys=False, ), "utf-8", ) return if template: params = dict(param) # Cannot be used with system try: template_obj = load_template(template) except LoadTemplateError as ex: raise click.ClickException(str(ex)) if not (extract or extract_last): extract = template_obj.extract extract_last = template_obj.extract_last # Combine with template fragments/system_fragments if template_obj.fragments: fragments = [*template_obj.fragments, *fragments] if template_obj.system_fragments: system_fragments = [*template_obj.system_fragments, *system_fragments] if template_obj.schema_object: schema = template_obj.schema_object tools, python_tools = _merge_template_tools(template_obj, tools, python_tools) if template_obj.options: options = _merge_template_options(template_obj, options) if "input" in template_obj.vars(): prompt = read_prompt() prompt, system = _apply_template(template_obj, prompt, params, system) if model_id is None and template_obj.model: model_id = template_obj.model attachments, attachment_types = _merge_template_attachments( template_obj, attachments, attachment_types ) if extract or extract_last or json_output: no_stream = True conversation = None if conversation_id or _continue: # Load the conversation - loads most recent if no ID provided try: conversation = load_conversation( conversation_id, async_=async_, database=database ) except UnknownModelError as ex: raise click.ClickException(str(ex)) if conversation_tools := _get_conversation_tools(conversation, tools): tools = conversation_tools # Figure out which model we are using if model_id is None: if conversation: model_id = conversation.model.model_id else: model_id = get_default_model() # Now resolve the model try: if async_: model = get_async_model(model_id) else: model = get_model(model_id) except UnknownModelError as ex: raise click.ClickException(ex) if show_model_options: click.echo(render_model_with_options(model_id, async_=async_)) return if conversation is None: # Always work through a conversation, even for a one-off prompt. # The legacy logger invents one anyway and throws the id away; # creating it here means both writers agree on which conversation # (and so which thread) this response belongs to. conversation = model.conversation() if conversation: # To ensure it can see the key conversation.model = model # Validate options validated_options = {} if options: # Validate with pydantic try: validated_options = { key: value for key, value in model.Options(**dict(options)) if value is not None } except pydantic.ValidationError as ex: raise click.ClickException(render_errors(ex.errors())) # Add on any default model options default_options = get_model_options(model.model_id) for key_, value in default_options.items(): if key_ not in validated_options: validated_options[key_] = value kwargs = {} resolved_attachments = [*attachments, *attachment_types] should_stream = model.can_stream and not no_stream if not should_stream: kwargs["stream"] = False if isinstance(model, (KeyModel, AsyncKeyModel)): kwargs["key"] = key prompt = read_prompt() response = None try: fragments_and_attachments = resolve_fragments( db, fragments, allow_attachments=True ) resolved_fragments = [ fragment for fragment in fragments_and_attachments if isinstance(fragment, Fragment) ] resolved_attachments.extend( attachment for attachment in fragments_and_attachments if isinstance(attachment, Attachment) ) resolved_system_fragments = resolve_fragments(db, system_fragments) except FragmentNotFound as ex: raise click.ClickException(str(ex)) prompt_method = model.prompt if conversation: prompt_method = conversation.prompt tool_kwargs = _tool_chain_kwargs( tools, python_tools, tools_debug, tools_approve, chain_limit, model=model ) if tool_kwargs: prompt_method = conversation.chain kwargs["options"] = validated_options kwargs.update(tool_kwargs) else: # Merge in options for the .prompt() methods kwargs.update(validated_options) if hide_reasoning: kwargs["hide_reasoning"] = True try: if async_: async def inner(): if should_stream: response = prompt_method( prompt, attachments=resolved_attachments, system=system, schema=schema, fragments=resolved_fragments, system_fragments=resolved_system_fragments, **kwargs, ) await display_async_stream_events( response.astream_events(), show_reasoning=not hide_reasoning, ) print() else: response = prompt_method( prompt, fragments=resolved_fragments, attachments=resolved_attachments, schema=schema, system=system, system_fragments=resolved_system_fragments, **kwargs, ) text = await response.text() if extract or extract_last: text = ( extract_fenced_code_block(text, last=extract_last) or text ) if not json_output: print(text) return response response = asyncio.run(inner()) else: response = prompt_method( prompt, fragments=resolved_fragments, attachments=resolved_attachments, system=system, schema=schema, system_fragments=resolved_system_fragments, **kwargs, ) if should_stream: display_stream_events( response.stream_events(), show_reasoning=not hide_reasoning, ) print() else: text = response.text() if extract or extract_last: text = extract_fenced_code_block(text, last=extract_last) or text if not json_output: print(text) # List of exceptions that should never be raised in pytest: except (ValueError, NotImplementedError) as ex: raise click.ClickException(str(ex)) except Exception as ex: # All other exceptions should raise in pytest, show to user otherwise if getattr(sys, "_called_from_test", False) or os.environ.get( "LLM_RAISE_ERRORS", None ): raise raise click.ClickException(str(ex)) if usage: if isinstance(response, ChainResponse): responses = response._responses else: responses = [response] for response_object in responses: # Show token usage to stderr in yellow click.echo( click.style( f"Token usage: {response_object.token_usage()}", fg="yellow", bold=True, ), err=True, ) # Log responses to the database log_db = None if (logs_on() or log) and not no_log: log_db = db elif json_output: # --json needs logged rows, so use a temporary in-memory database log_db = sqlite_utils.Database(memory=True) migrate(log_db) if log_db is not None: # Could be Response, AsyncResponse, ChainResponse, AsyncChainResponse if isinstance(response, AsyncResponse): response = asyncio.run(response.to_sync_response()) # At this point ALL forms should have a log_to_db() method that works: response.log_to_db(log_db) if json_output: if isinstance(response, _BaseChainResponse): response_ids = [response_.id for response_ in response._responses] else: response_ids = [response.id] click.echo(logs_json_for_response_ids(log_db, response_ids)) @cli.command() @click.option("-s", "--system", help="System prompt to use") @click.option("model_id", "-m", "--model", help="Model to use", envvar="LLM_MODEL") @click.option( "_continue", "-c", "--continue", is_flag=True, flag_value=-1, help="Continue the most recent conversation.", ) @click.option( "conversation_id", "--cid", "--conversation", help="Continue the conversation with the given ID.", ) @click.option( "fragments", "-f", "--fragment", multiple=True, help="Fragment (alias, URL, hash or file path) to add to the prompt", ) @click.option( "system_fragments", "--sf", "--system-fragment", multiple=True, help="Fragment to add to system prompt", ) @click.option("-t", "--template", help="Template to use") @click.option( "-p", "--param", multiple=True, type=(str, str), help="Parameters for template", ) @click.option( "options", "-o", "--option", type=(str, str), multiple=True, help="key/value options for the model", ) @click.option( "-d", "--database", type=click.Path(readable=True, dir_okay=False), help="Path to log database", ) @click.option("--no-stream", is_flag=True, help="Do not stream output") @click.option("-R", "--hide-reasoning", is_flag=True, help="Hide reasoning output") @click.option("--key", help="API key to use") @tool_options def chat( system, model_id, _continue, conversation_id, fragments, system_fragments, template, param, options, no_stream, hide_reasoning, key, database, tools, python_tools, tools_debug, tools_approve, chain_limit, ): """ Hold an ongoing chat with a model. """ # Left and right arrow keys to move cursor: if sys.platform != "win32": readline.parse_and_bind("\\e[D: backward-char") readline.parse_and_bind("\\e[C: forward-char") else: readline.parse_and_bind("bind -x '\\e[D: backward-char'") readline.parse_and_bind("bind -x '\\e[C: forward-char'") log_path = pathlib.Path(database) if database else logs_db_path() (log_path.parent).mkdir(parents=True, exist_ok=True) db = sqlite_utils.Database(log_path) migrate(db) conversation = None if conversation_id or _continue: # Load the conversation - loads most recent if no ID provided try: conversation = load_conversation(conversation_id, database=database) except UnknownModelError as ex: raise click.ClickException(str(ex)) if conversation_tools := _get_conversation_tools(conversation, tools): tools = conversation_tools template_obj = None if template: params = dict(param) try: template_obj = load_template(template) except LoadTemplateError as ex: raise click.ClickException(str(ex)) if model_id is None and template_obj.model: model_id = template_obj.model tools, python_tools = _merge_template_tools(template_obj, tools, python_tools) # Figure out which model we are using if model_id is None: if conversation: model_id = conversation.model.model_id else: model_id = get_default_model() # Now resolve the model try: model = get_model(model_id) except KeyError: raise click.ClickException(f"'{model_id}' is not a known model") if conversation is None: # Start a fresh conversation for this chat conversation = Conversation(model=model) else: # Ensure it can see the API key conversation.model = model # Validate options validated_options = get_model_options(model.model_id) if options: try: validated_options = { key: value for key, value in model.Options(**dict(options)) if value is not None } except pydantic.ValidationError as ex: raise click.ClickException(render_errors(ex.errors())) kwargs = {} if validated_options: kwargs["options"] = validated_options kwargs.update( _tool_chain_kwargs( tools, python_tools, tools_debug, tools_approve, chain_limit, model=model, ) ) should_stream = model.can_stream and not no_stream if not should_stream: kwargs["stream"] = False if key and isinstance(model, KeyModel): kwargs["key"] = key if hide_reasoning: kwargs["hide_reasoning"] = True try: fragments_and_attachments = resolve_fragments( db, fragments, allow_attachments=True ) argument_fragments = [ fragment for fragment in fragments_and_attachments if isinstance(fragment, Fragment) ] argument_attachments = [ attachment for attachment in fragments_and_attachments if isinstance(attachment, Attachment) ] argument_system_fragments = resolve_fragments(db, system_fragments) except FragmentNotFound as ex: raise click.ClickException(str(ex)) def transform_chat_prompt(prompt): nonlocal system if template_obj: prompt, system = _apply_template(template_obj, prompt, params, system) return prompt def execute_chat_prompt(prompt, fragments, attachments): nonlocal system, argument_system_fragments response = conversation.chain( prompt, fragments=fragments, system_fragments=argument_system_fragments, attachments=attachments, system=system, **kwargs, ) # System prompt and system fragments only sent for the first message system = None argument_system_fragments = [] return response _run_chat( model.model_id, execute_chat_prompt, db=db, initial_fragments=argument_fragments, initial_attachments=argument_attachments, transform_prompt=transform_chat_prompt, after_response=lambda response: response.log_to_db(db), show_reasoning=not hide_reasoning, ) def load_conversation( conversation_id: str | None, async_=False, database=None, ) -> _BaseConversation | None: log_path = pathlib.Path(database) if database else logs_db_path() db = sqlite_utils.Database(log_path) migrate(db) if conversation_id is None: # Most recent conversation from either generation of tables - # thread ids are conversation ids, so the union dedupes rows # from the dual-write era. matches = list(db.query(""" select id from ( select id from threads union select id from conversations ) order by id desc limit 1 """)) if matches: conversation_id = matches[0]["id"] else: return None try: row = cast(sqlite_utils.db.Table, db["conversations"]).get(conversation_id) except sqlite_utils.db.NotFoundError: # No legacy record - reconstruct the equivalent from the thread # and its most recent turn's model. try: thread_row = cast(sqlite_utils.db.Table, db["threads"]).get(conversation_id) except sqlite_utils.db.NotFoundError: raise click.ClickException( f"No conversation found with id={conversation_id}" ) model_match = next( db.query( "select model from turns where thread_id = ? order by id desc limit 1", [conversation_id], ), None, ) if model_match is None: raise click.ClickException( f"No conversation found with id={conversation_id}" ) row = { "id": conversation_id, "name": thread_row["name"], "model": model_match["model"], } # Inflate that conversation conversation_class = AsyncConversation if async_ else Conversation response_class = AsyncResponse if async_ else Response conversation = conversation_class.from_row(row) for response in db["responses"].rows_where( "conversation_id = ?", [conversation_id], order_by="id" ): response_obj = response_class.from_row(db, response) if conversation.responses: previous_response = conversation.responses[-1] # SQLite rows store each response's legacy current-turn inputs # (prompt text, attachments, tool_results), not the full # prompt.messages chain. Rebuild that chain here so follow-up # prompts via `llm -c` satisfy the Prompt.messages invariant. response_obj.prompt._explicit_messages = ( list(previous_response.prompt.messages) + list(previous_response._messages_now()) + list(response_obj.prompt.messages) ) conversation.responses.append(response_obj) # If this conversation has a thread in the content-addressed tables, # take the history from there. That chain is the exact message list # that was sent and returned, so reasoning signatures and provider # metadata survive - unlike the rebuild above, which can only work # from the flattened legacy columns. try: conversation.loaded_messages = LogStore(db).thread_messages(conversation_id) except KeyError: pass # Plugin and server-side tools recorded against the first turn, for # the same reuse-on-continue behaviour the rebuilt responses provide. # Configured instances are collapsed into a single spec string like # Datasette({"url": "..."}) - the same format -T accepts - so the # instance can be reconstructed with its configuration. loaded_tools = [] seen_instance_ids = set() supported_server_side_tool_names = { tool_class.__name__ for tool_class in conversation.model.supported_server_side_tools } for tool_row in db.query( """ select tools.name, tools.plugin, turn_tools.instance_id, tool_instances.name as instance_name, tool_instances.arguments as instance_arguments from tools join turn_tools on turn_tools.tool_id = tools.id left join tool_instances on tool_instances.id = turn_tools.instance_id where turn_tools.turn_id = ( select id from turns where thread_id = ? order by id limit 1 ) """, [conversation_id], ): if ( tool_row["plugin"] is None and tool_row["instance_name"] not in supported_server_side_tool_names ): continue if tool_row["instance_id"] is None: loaded_tools.append(tool_row["name"]) elif tool_row["instance_id"] not in seen_instance_ids: seen_instance_ids.add(tool_row["instance_id"]) arguments = tool_row["instance_arguments"] if arguments and arguments != "{}": loaded_tools.append( "{}({})".format(tool_row["instance_name"], arguments) ) else: loaded_tools.append(tool_row["instance_name"]) conversation.loaded_tools = loaded_tools return conversation @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def keys(): "Manage stored API keys for different models" @keys.command(name="list") def keys_list(): "List names of all stored keys" path = user_dir() / "keys.json" if not path.exists(): click.echo("No keys found") return keys = json.loads(path.read_text()) for key in sorted(keys.keys()): if key != "// Note": click.echo(key) @keys.command(name="path") def keys_path_command(): "Output the path to the keys.json file" click.echo(user_dir() / "keys.json") @keys.command(name="get") @click.argument("name") def keys_get(name): """ Return the value of a stored key Example usage: \b export OPENAI_API_KEY=$(llm keys get openai) """ path = user_dir() / "keys.json" if not path.exists(): raise click.ClickException("No keys found") keys = json.loads(path.read_text()) try: click.echo(keys[name]) except KeyError: raise click.ClickException(f"No key found with name '{name}'") @keys.command(name="set") @click.argument("name") @click.option("--value", prompt="Enter key", hide_input=True, help="Value to set") def keys_set(name, value): """ Save a key in the keys.json file Example usage: \b $ llm keys set openai Enter key: ... """ default = {"// Note": "This file stores secret API credentials. Do not share!"} path = user_dir() / "keys.json" path.parent.mkdir(parents=True, exist_ok=True) if not path.exists(): path.write_text(json.dumps(default)) path.chmod(0o600) try: current = json.loads(path.read_text()) except json.decoder.JSONDecodeError: current = default current[name] = value path.write_text(json.dumps(current, indent=2) + "\n") @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def logs(): "Tools for exploring logged prompts and responses" @logs.command(name="path") def logs_path(): "Output the path to the logs.db file" click.echo(logs_db_path()) @logs.command(name="status") def logs_status(): "Show current status of database logging" path = logs_db_path() if not path.exists(): click.echo(f"No log database found at {path}") return if logs_on(): click.echo("Logging is ON for all prompts".format()) else: click.echo("Logging is OFF".format()) db = sqlite_utils.Database(path) migrate(db) click.echo(f"Found log database at {path}") click.echo("Number of threads logged:\t{}".format(db["threads"].count)) click.echo("Number of turns logged:\t\t{}".format(db["turns"].count)) legacy_conversations = db["conversations"].count legacy_responses = db["responses"].count if legacy_conversations or legacy_responses: click.echo(f"Number of legacy conversations:\t{legacy_conversations}") click.echo(f"Number of legacy responses:\t{legacy_responses}") click.echo(f"Database file size: \t\t{_human_readable_size(path.stat().st_size)}") @logs.command(name="backup") @click.argument("path", type=click.Path(dir_okay=True, writable=True)) def backup(path): "Backup your logs database to this file" logs_path = logs_db_path() path = pathlib.Path(path) db = sqlite_utils.Database(logs_path) try: db.execute("vacuum into ?", [str(path)]) except Exception as ex: # noqa: BLE001 raise click.ClickException(str(ex)) click.echo(f"Backed up {_human_readable_size(path.stat().st_size)} to {path}") @logs.command(name="on") def logs_turn_on(): "Turn on logging for all prompts" path = user_dir() / "logs-off" if path.exists(): path.unlink() @logs.command(name="off") def logs_turn_off(): "Turn off logging for all prompts" path = user_dir() / "logs-off" path.touch() def annotate_log_rows(db, rows, expand=False, truncate=False): """ Modify log rows from the merged reader in place: attach fragments and tool information, decode (or, if truncate is on, remove) their JSON columns and strip the reader's internal keys. Returns a dict mapping row id to its attachments, for log_rows_as_json and the rendered output. """ store = LogStore(db) # New rows carry their extras in the row's parts; legacy rows # batch-fetch from the legacy tables. legacy_extras = legacy_log_row_extras( db, [row["id"] for row in rows if row.get("_legacy")] ) extras_by_id = { row["id"]: ( legacy_extras[row["id"]] if row.get("_legacy") else log_row_extras(store, row) ) for row in rows } for row in rows: for internal in ( "_input_parts", "_output_parts", "_parent_message_hash", "_input_message_hashes", "_tip_message_hash", "_legacy", "_search_rank", ): row.pop(internal, None) extras = extras_by_id[row["id"]] if truncate: row["prompt"] = truncate_string(row["prompt"] or "") row["response"] = truncate_string(row["response"] or "") # Add prompt and system fragments for key in ("prompt_fragments", "system_fragments"): row[key] = [ { "hash": fragment["hash"], "content": ( fragment["content"] if expand else truncate_string(fragment["content"]) ), "aliases": json.loads(fragment["aliases"]), } for fragment in extras[key] ] # Either decode or remove all JSON keys keys = list(row.keys()) for key in keys: if key.endswith("_json") and row[key] is not None: if truncate: del row[key] else: row[key] = json.loads(row[key]) row.update( { "tools": extras["tools"], "tool_calls": extras["tool_calls"], "tool_results": extras["tool_results"], } ) return {id: extras["attachments"] for id, extras in extras_by_id.items()} def log_rows_as_json(rows, attachments_by_id): "Serialize annotated log rows to the JSON used by 'llm logs --json'" for row in rows: row["attachments"] = [ {k: v for k, v in attachment.items() if k != "response_id"} for attachment in attachments_by_id.get(row["id"], []) ] return json.dumps(list(rows), indent=2) def logs_json_for_response_ids(db, ids): """ Return the JSON that 'llm logs --json' would output for these response IDs, in chronological order """ if not ids: return "[]" rows = merged_log_rows(LogStore(db), ids=list(ids)) # Newest first out of the reader, chronological out here rows.reverse() return log_rows_as_json(rows, annotate_log_rows(db, rows)) @logs.command(name="list") @click.option( "-n", "--count", type=int, default=None, help="Number of entries to show - defaults to 3, use 0 for all", ) @click.option( "-p", "--path", type=click.Path(readable=True, exists=True, dir_okay=False), help="Path to log database", hidden=True, ) @click.option( "-d", "--database", type=click.Path(readable=True, exists=True, dir_okay=False), help="Path to log database", ) @click.option("-m", "--model", help="Filter by model or model alias") @click.option("-q", "--query", help="Search for logs matching this string") @click.option( "fragments", "--fragment", "-f", help="Filter for prompts using these fragments", multiple=True, ) @click.option( "tools", "-T", "--tool", multiple=True, help="Filter for prompts with results from these tools", ) @click.option( "any_tools", "--tools", is_flag=True, help="Filter for prompts with results from any tools", ) @schema_option @click.option( "--schema-multi", help="JSON schema used for multiple results", ) @click.option( "-l", "--latest", is_flag=True, help="Return latest results matching search query" ) @click.option( "--data", is_flag=True, help="Output newline-delimited JSON data for schema" ) @click.option("--data-array", is_flag=True, help="Output JSON array of data for schema") @click.option("--data-key", help="Return JSON objects from array in this key") @click.option( "--data-ids", is_flag=True, help="Attach corresponding IDs to JSON objects" ) @click.option("-t", "--truncate", is_flag=True, help="Truncate long strings in output") @click.option( "-s", "--short", is_flag=True, help="Shorter YAML output with truncated prompts" ) @click.option("-u", "--usage", is_flag=True, help="Include token usage") @click.option("-r", "--response", is_flag=True, help="Just output the last response") @click.option("-x", "--extract", is_flag=True, help="Extract first fenced code block") @click.option( "extract_last", "--xl", "--extract-last", is_flag=True, help="Extract last fenced code block", ) @click.option( "current_conversation", "-c", "--current", is_flag=True, flag_value=-1, help="Show logs from the current conversation", ) @click.option( "conversation_id", "--cid", "--conversation", help="Show logs for this conversation ID", ) @click.option("--id-gt", help="Return responses with ID > this") @click.option("--id-gte", help="Return responses with ID >= this") @click.option( "json_output", "--json", is_flag=True, help="Output logs as JSON", ) @click.option( "--expand", "-e", is_flag=True, help="Expand fragments to show their content", ) def logs_list( count, path, database, model, query, fragments, tools, any_tools, schema_input, schema_multi, latest, data, data_array, data_key, data_ids, truncate, short, usage, response, extract, extract_last, current_conversation, conversation_id, id_gt, id_gte, json_output, expand, ): "Show logged prompts and their responses" if database and not path: path = database path = pathlib.Path(path or logs_db_path()) if not path.exists(): raise click.ClickException(f"No log database found at {path}") db = sqlite_utils.Database(path) migrate(db) if schema_multi: schema_input = schema_multi schema = resolve_schema_input(db, schema_input, load_template) if schema_multi: schema = multi_schema(schema) if short and (json_output or response): invalid = " or ".join( [ flag[0] for flag in (("--json", json_output), ("--response", response)) if flag[1] ] ) raise click.ClickException(f"Cannot use --short and {invalid} together") if response and not current_conversation and not conversation_id: current_conversation = True if current_conversation: try: # Thread ids are conversation ids and both id spaces are # ULIDs, so the most recent of either world wins. conversation_id = next(db.query(""" select conversation_id from ( select thread_id as conversation_id, id from turns union all select conversation_id, id from responses ) order by id desc limit 1 """))["conversation_id"] except StopIteration: # No conversations yet raise click.ClickException("No conversations found") # For --conversation set limit 0, if not explicitly set if count is None: if conversation_id: count = 0 else: count = 3 model_id = None if model: # Resolve alias, if any try: model_id = get_model(model).model_id except UnknownModelError: # Maybe they uninstalled a model, use the -m option as-is model_id = model fragment_hashes = [fragment.id() for fragment in resolve_fragments(db, fragments)] schema_id = make_schema_id(schema)[0] if schema else None store = LogStore(db) try: rows = merged_log_rows( store, count=count if count and count > 0 else None, model_id=model_id, thread_id=conversation_id, fragment_hashes=fragment_hashes, tool_names=tools, any_tools=any_tools, schema_id=schema_id, id_gt=id_gt, id_gte=id_gte, query=query, latest=latest, ) except sqlite3.OperationalError as ex: if query: # Almost certainly FTS5 syntax - unbalanced quotes, stray # operators and the like raise click.ClickException( f"Invalid search query: {ex} - see the FTS5 query syntax " "documentation at https://sqlite.org/fts5.html#full_text_query_syntax" ) raise # Newest first out of the query, but read chronologically - except # for search results, which are already most-relevant first. if not query and not data: rows.reverse() if data or data_array or data_key or data_ids: # Special case for --data to output valid JSON to_output = [] for row in rows: response = row["response"] or "" try: decoded = json.loads(response) if ( isinstance(decoded, dict) and (data_key in decoded) and all(isinstance(item, dict) for item in decoded[data_key]) ): new_items = list(decoded[data_key]) else: new_items = [decoded] if data_ids: for item in new_items: item[find_unused_key(item, "response_id")] = row["id"] item[find_unused_key(item, "conversation_id")] = row["id"] to_output.extend(new_items) except ValueError: pass for line in output_rows_as_json(to_output, nl=not data_array, compact=True): click.echo(line) return attachments_by_id = annotate_log_rows(db, rows, expand=expand, truncate=truncate) output = None if json_output: # Output as JSON if requested output = log_rows_as_json(rows, attachments_by_id) elif extract or extract_last: # Extract and return first code block for row in rows: output = extract_fenced_code_block(row["response"], last=extract_last) if output is not None: break elif response and rows: # Just output the last response output = rows[-1]["response"] if output is not None: click.echo(output) else: # Output neatly formatted human-readable logs def _fenced_block(value): # Fenced code block, indented to nest inside a list item num_backticks = 3 while "`" * num_backticks in value: num_backticks += 1 fence = "`" * num_backticks return textwrap.indent(f"{fence}\n{value}\n{fence}", " ") def _inline_code(value): num_backticks = 1 while "`" * num_backticks in value: num_backticks += 1 delimiter = "`" * num_backticks if value.startswith("`") or value.endswith("`"): return f"{delimiter} {value} {delimiter}" return f"{delimiter}{value}{delimiter}" def _format_tool_call_arguments(arguments): if not isinstance(arguments, dict) or not arguments: return f" Arguments: {_inline_code(json.dumps(arguments))}" lines = [] for key, value in arguments.items(): if isinstance(value, str): lines.append(f" {key}:") lines.append(_fenced_block(value)) else: lines.append(f" {key}: {_inline_code(json.dumps(value))}") return "\n".join(lines) def _token_usage_markdown(input_tokens, output_tokens, token_details): usage = token_usage_string(input_tokens, output_tokens, None) if token_details: details = _inline_code(json.dumps(token_details)) if usage: return f"{usage}, {details}" return details return usage def _display_fragments(fragments, title): if not fragments: return if not expand: content = "\n".join( ["- {}".format(fragment["hash"]) for fragment in fragments] ) else: #
for each one bits = [] for fragment in fragments: bits.append( "
{}\n{}\n
".format( fragment["hash"], maybe_fenced_code(fragment["content"]) ) ) content = "\n".join(bits) click.echo(f"\n### {title}\n\n{content}") current_system = None should_show_conversation = True seen_tool_hashes = set() for row in rows: if short: system = truncate_string( row["system"] or "", 120, normalize_whitespace=True ) prompt = truncate_string( row["prompt"] or "", 120, normalize_whitespace=True, keep_end=True ) cid = row["conversation_id"] attachments = attachments_by_id.get(row["id"]) obj = { "model": row["model"], "datetime": row["datetime_utc"].split(".")[0], "conversation": cid, } if row["tool_calls"]: obj["tool_calls"] = [ "{}({})".format( tool_call["name"], json.dumps(tool_call["arguments"]) ) for tool_call in row["tool_calls"] ] if row["tool_results"]: obj["tool_results"] = [ "{}: {}".format( tool_result["name"], truncate_string(tool_result["output"]) ) for tool_result in row["tool_results"] ] if system: obj["system"] = system if prompt: obj["prompt"] = prompt if attachments: items = [] for attachment in attachments: details = {"type": attachment["type"]} if attachment.get("path"): details["path"] = attachment["path"] if attachment.get("url"): details["url"] = attachment["url"] items.append(details) obj["attachments"] = items for key in ("prompt_fragments", "system_fragments"): obj[key] = [fragment["hash"] for fragment in row[key]] if usage and (row["input_tokens"] or row["output_tokens"]): usage_details = { "input": row["input_tokens"], "output": row["output_tokens"], } if row["token_details"]: usage_details["details"] = json.loads(row["token_details"]) obj["usage"] = usage_details click.echo(yaml.dump([obj], sort_keys=False).strip()) continue # Not short, output Markdown click.echo( "# {}{}\n{}".format( row["datetime_utc"].split(".")[0], ( " conversation: {} id: {}".format( row["conversation_id"], row["id"] ) if should_show_conversation else "" ), ( ( "\nModel: **{}**{}\n".format( row["model"], ( " (resolved: **{}**)".format(row["resolved_model"]) if row["resolved_model"] else "" ), ) ) if should_show_conversation else "" ), ) ) # In conversation log mode only show it for the first one if conversation_id: should_show_conversation = False click.echo("## Prompt\n\n{}".format(row["prompt"] or "-- none --")) _display_fragments(row["prompt_fragments"], "Prompt fragments") if row["options_json"]: options = row["options_json"] if isinstance(options, str): options = json.loads(options) if options: options_text = "\n".join( f"- {key}: {value}" for key, value in options.items() ) click.echo(f"\n## Options\n\n{options_text}") if row["system"] != current_system: if row["system"] is not None: click.echo("\n## System\n\n{}".format(row["system"])) current_system = row["system"] _display_fragments(row["system_fragments"], "System fragments") if row["schema_json"]: click.echo( "\n## Schema\n\n```json\n{}\n```".format( json.dumps(row["schema_json"], indent=2) ) ) # Show tool calls and results if row["tools"]: click.echo("\n### Tools\n") def echo_tool(tool, indent=""): if tool["hash"] in seen_tool_hashes: block = "- **{}**: `{}`".format(tool["name"], tool["hash"][:7]) else: seen_tool_hashes.add(tool["hash"]) block = "- **{}**: `{}` \n{} \n Arguments: `{}`".format( tool["name"], tool["hash"], textwrap.indent( (tool["description"] or "").rstrip(), " " ), json.dumps(tool["input_schema"].get("properties", {})), ) click.echo(textwrap.indent(block, indent)) # Tools provided by the same configured toolbox instance # nest beneath one instance line rather than repeating it plain_tools = [] by_instance: dict = {} for tool in row["tools"]: instance = tool.get("instance") if instance: key = (instance["name"], instance["arguments"]) by_instance.setdefault(key, []).append(tool) else: plain_tools.append(tool) for tool in plain_tools: echo_tool(tool) for (name, arguments), instance_tools in by_instance.items(): click.echo( "- `{}({})`:".format( name, arguments if arguments and arguments != "{}" else "", ) ) for tool in instance_tools: echo_tool(tool, " ") if row["tool_results"]: click.echo("\n### Tool results\n") for tool_result in row["tool_results"]: attachments = "" for attachment in tool_result["attachments"]: desc = "" if attachment.get("type"): desc += attachment["type"] + ": " if attachment.get("path"): desc += attachment["path"] elif attachment.get("url"): desc += attachment["url"] elif attachment.get("content"): desc += f"<{attachment['content_length']:,} bytes>" attachments += f"\n - {desc}" click.echo( "- **{}**: `{}` \n{}{}{}".format( tool_result["name"], tool_result["tool_call_id"], _fenced_block(tool_result["output"]), ( " \n **Error**: {}\n".format( tool_result["exception"] ) if tool_result["exception"] else "" ), attachments, ) ) attachments = attachments_by_id.get(row["id"]) if attachments: click.echo("\n### Attachments\n") for i, attachment in enumerate(attachments, 1): if attachment["path"]: path = attachment["path"] click.echo( "{}. **{}**: `{}`".format(i, attachment["type"], path) ) elif attachment["url"]: click.echo( "{}. **{}**: {}".format( i, attachment["type"], attachment["url"] ) ) elif attachment["content_length"]: click.echo( "{}. **{}**: `<{} bytes>`".format( i, attachment["type"], f"{attachment['content_length']:,}", ) ) # If a schema was provided and the row is valid JSON, pretty print and syntax highlight it response = row["response"] if row["schema_json"]: try: parsed = json.loads(response) response = f"```json\n{json.dumps(parsed, indent=2)}\n```" except ValueError: pass if row.get("reasoning"): click.echo("\n## Reasoning\n\n{}".format(row["reasoning"].rstrip())) click.echo("\n## Response\n") if row["tool_calls"]: click.echo("### Tool calls\n") for tool_call in row["tool_calls"]: click.echo( "- **{}**: `{}` \n{}".format( tool_call["name"], tool_call["tool_call_id"], _format_tool_call_arguments(tool_call["arguments"]), ) ) click.echo("") if response: click.echo(f"{response}\n") if usage: token_usage = _token_usage_markdown( row["input_tokens"], row["output_tokens"], json.loads(row["token_details"]) if row["token_details"] else None, ) if token_usage: click.echo(f"## Token usage\n\n{token_usage}\n") @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def models(): "Manage available models" _type_lookup = { "number": "float", "integer": "int", "string": "str", "object": "dict", } def model_matches_id_or_alias(model_with_aliases, model_ids): ids_and_aliases = set( [model_with_aliases.model.model_id] + model_with_aliases.aliases ) return ids_and_aliases.intersection(model_ids) def render_model_with_aliases( model_with_aliases, *, options=False, async_=False, models_that_have_shown_options=None, ): extra_info = [] if model_with_aliases.aliases: extra_info.append("aliases: {}".format(", ".join(model_with_aliases.aliases))) model = model_with_aliases.model if not async_ else model_with_aliases.async_model output = str(model) if extra_info: output += " ({})".format(", ".join(extra_info)) if options and model.Options.model_json_schema()["properties"]: output += "\n Options:" for name, field in model.Options.model_json_schema()["properties"].items(): any_of = field.get("anyOf") if any_of is None: any_of = [{"type": field.get("type", "str")}] types = ", ".join( [ _type_lookup.get(item.get("type"), item.get("type", "str")) for item in any_of if item.get("type") != "null" ] ) bits = ["\n ", name, ": ", types] description = field.get("description", "") if ( description and models_that_have_shown_options is not None and model.__class__ not in models_that_have_shown_options ): wrapped = textwrap.wrap(description, 70) bits.append("\n ") bits.extend("\n ".join(wrapped)) output += "".join(bits) if models_that_have_shown_options is not None: models_that_have_shown_options.add(model.__class__) if options and model.attachment_types: attachment_types = ", ".join(sorted(model.attachment_types)) wrapper = textwrap.TextWrapper( width=min(max(shutil.get_terminal_size().columns, 30), 70), initial_indent=" ", subsequent_indent=" ", ) output += f"\n Attachment types:\n{wrapper.fill(attachment_types)}" features = ( [] + (["streaming"] if model.can_stream else []) + (["schemas"] if model.supports_schema else []) + (["tools"] if model.supports_tools else []) + (["async"] if model_with_aliases.async_model else []) ) if options and features: output += "\n Features:\n{}".format( "\n".join(f" - {feature}" for feature in features) ) if options and hasattr(model, "needs_key") and model.needs_key: output += "\n Keys:" if hasattr(model, "needs_key") and model.needs_key: output += f"\n key: {model.needs_key}" if hasattr(model, "key_env_var") and model.key_env_var: output += f"\n env_var: {model.key_env_var}" return output def render_model_with_options(model_id, *, async_=False): for model_with_aliases in get_models_with_aliases(): if model_matches_id_or_alias(model_with_aliases, [model_id]): return render_model_with_aliases( model_with_aliases, options=True, async_=async_, models_that_have_shown_options=set(), ) raise click.ClickException(f"'{model_id}' is not a known model") @models.command(name="list") @click.option( "--options", is_flag=True, help="Show options for each model, if available" ) @click.option("async_", "--async", is_flag=True, help="List async models") @click.option("--schemas", is_flag=True, help="List models that support schemas") @click.option("--tools", is_flag=True, help="List models that support tools") @click.option("json_", "--json", is_flag=True, help="Output as JSON") @click.option( "-q", "--query", multiple=True, help="Search for models matching these strings", ) @click.option("model_ids", "-m", "--model", help="Specific model IDs", multiple=True) def models_list(options, async_, schemas, tools, json_, query, model_ids): "List available models" models_that_have_shown_options = set() json_models = [] for model_with_aliases in get_models_with_aliases(): if async_ and not model_with_aliases.async_model: continue # Only show models where every provided query string matches if query and not all(model_with_aliases.matches(q) for q in query): continue if model_ids and not model_matches_id_or_alias(model_with_aliases, model_ids): continue if schemas and not model_with_aliases.model.supports_schema: continue if tools and not model_with_aliases.model.supports_tools: continue if json_: model = ( model_with_aliases.async_model if async_ else model_with_aliases.model ) model_json = { "model_id": model.model_id, "aliases": model_with_aliases.aliases, "can_stream": model.can_stream, "supports_schema": model.supports_schema, "supports_tools": model.supports_tools, "supports_async": model_with_aliases.async_model is not None, "attachment_types": sorted(model.attachment_types), "server_side_tools": [ { "name": tool_class.__name__, "plugin": getattr(tool_class, "plugin", None), } for tool_class in model.supported_server_side_tools ], } if options: model_json["options"] = model.Options.model_json_schema()["properties"] json_models.append(model_json) continue click.echo( render_model_with_aliases( model_with_aliases, options=options, async_=async_, models_that_have_shown_options=models_that_have_shown_options, ) ) if json_: click.echo(json.dumps(json_models, indent=2)) return if not query and not options and not schemas and not model_ids: click.echo(f"Default: {get_default_model()}") @models.command(name="default") @click.argument("model", required=False) def models_default(model): "Show or set the default model" if not model: click.echo(get_default_model()) return # Validate it is a known model try: model = get_model(model) set_default_model(model.model_id) except KeyError: raise click.ClickException(f"Unknown model: {model}") @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def templates(): "Manage stored prompt templates" @templates.command(name="list") def templates_list(): "List available prompt templates" path = template_dir() pairs = [] for file in path.glob("*.yaml"): name = file.stem try: template = load_template(name) except LoadTemplateError: # Skip invalid templates continue text = [] if template.system: text.append(f"system: {template.system}") if template.prompt: text.append(f" prompt: {template.prompt}") else: text = [template.prompt if template.prompt else ""] pairs.append((name, "".join(text).replace("\n", " "))) try: max_name_len = max(len(p[0]) for p in pairs) except ValueError: return else: fmt = "{name:<" + str(max_name_len) + "} : {prompt}" for name, prompt in sorted(pairs): text = fmt.format(name=name, prompt=prompt) click.echo(display_truncated(text)) @templates.command(name="show") @click.argument("name") def templates_show(name): "Show the specified prompt template" try: template = load_template(name) except LoadTemplateError: raise click.ClickException(f"Template '{name}' not found or invalid") click.echo( yaml.dump( {k: v for k, v in template.model_dump().items() if v is not None}, indent=4, default_flow_style=False, ) ) @templates.command(name="edit") @click.argument("name") def templates_edit(name): "Edit the specified prompt template using the default $EDITOR" # First ensure it exists path = template_dir() / f"{name}.yaml" if not path.exists(): path.write_text(DEFAULT_TEMPLATE, "utf-8") click.edit(filename=str(path)) # Validate that template load_template(name) @templates.command(name="path") def templates_path(): "Output the path to the templates directory" click.echo(template_dir()) @templates.command(name="loaders") def templates_loaders(): "Show template loaders registered by plugins" found = False for prefix, loader in get_template_loaders().items(): found = True docs = "Undocumented" if loader.__doc__: docs = textwrap.dedent(loader.__doc__).strip() click.echo(f"{prefix}:") click.echo(textwrap.indent(docs, " ")) if not found: click.echo("No template loaders found") @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def schemas(): "Manage stored schemas" @schemas.command(name="list") @click.option( "-p", "--path", type=click.Path(readable=True, exists=True, dir_okay=False), help="Path to log database", hidden=True, ) @click.option( "-d", "--database", type=click.Path(readable=True, exists=True, dir_okay=False), help="Path to log database", ) @click.option( "queries", "-q", "--query", multiple=True, help="Search for schemas matching this string", ) @click.option("--full", is_flag=True, help="Output full schema contents") @click.option("json_", "--json", is_flag=True, help="Output as JSON") @click.option("nl", "--nl", is_flag=True, help="Output as newline-delimited JSON") def schemas_list(path, database, queries, full, json_, nl): "List stored schemas" if database and not path: path = database path = pathlib.Path(path or logs_db_path()) if not path.exists(): raise click.ClickException(f"No log database found at {path}") db = sqlite_utils.Database(path) migrate(db) params = [] where_sql = "" if queries: where_bits = ["schemas.content like ?" for _ in queries] where_sql += " where {}".format(" and ".join(where_bits)) params.extend(f"%{q}%" for q in queries) sql = f""" select schemas.id, schemas.content, max(responses.datetime_utc) as recently_used, count(*) as times_used from schemas join responses on responses.schema_id = schemas.id {where_sql} group by responses.schema_id order by recently_used """ rows = db.query(sql, params) if json_ or nl: for line in output_rows_as_json(rows, json_cols={"content"}, nl=nl): click.echo(line) return for row in rows: click.echo("- id: {}".format(row["id"])) if full: click.echo( " schema: |\n{}".format( textwrap.indent( json.dumps(json.loads(row["content"]), indent=2), " " ) ) ) else: click.echo( " summary: |\n {}".format( schema_summary(json.loads(row["content"])) ) ) click.echo( " usage: |\n {} time{}, most recently {}".format( row["times_used"], "s" if row["times_used"] != 1 else "", row["recently_used"], ) ) @schemas.command(name="show") @click.argument("schema_id") @click.option( "-p", "--path", type=click.Path(readable=True, exists=True, dir_okay=False), help="Path to log database", hidden=True, ) @click.option( "-d", "--database", type=click.Path(readable=True, exists=True, dir_okay=False), help="Path to log database", ) def schemas_show(schema_id, path, database): "Show a stored schema" if database and not path: path = database path = pathlib.Path(path or logs_db_path()) if not path.exists(): raise click.ClickException(f"No log database found at {path}") db = sqlite_utils.Database(path) migrate(db) try: row = db["schemas"].get(schema_id) except sqlite_utils.db.NotFoundError: raise click.ClickException("Invalid schema ID") click.echo(json.dumps(json.loads(row["content"]), indent=2)) @schemas.command(name="dsl") @click.argument("input") @click.option("--multi", is_flag=True, help="Wrap in an array") def schemas_dsl_debug(input, multi): """ Convert LLM's schema DSL to a JSON schema \b llm schema dsl 'name, age int, bio: their bio' """ schema = schema_dsl(input, multi) click.echo(json.dumps(schema, indent=2)) @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def tools(): "Manage tools that can be made available to LLMs" @tools.command(name="list") @click.argument("tool_defs", nargs=-1) @click.option("json_", "--json", is_flag=True, help="Output as JSON") @click.option("model_id", "-m", "--model", help="List tools supported by this model") @click.option( "python_tools", "--functions", help="Python code block or file path defining functions to register as tools", multiple=True, ) def tools_list(tool_defs, json_, model_id, python_tools): "List available tools, optionally including tools supported by a model" model = None if model_id: try: model = get_model(model_id) except UnknownModelError as ex: raise click.ClickException(str(ex)) server_side_tools = [] if model is not None: for tool_class in model.supported_server_side_tools: try: signature = str(inspect.signature(tool_class)) except (ValueError, TypeError): signature = "(...)" server_side_tools.append( { "name": tool_class.__name__, "description": inspect.getdoc(tool_class), "signature": signature, "server_side": True, } ) def introspect_tools(toolbox): # Instances report their tools(), which may be generated dynamically. # Classes can only report tools for their introspectable methods. if isinstance(toolbox, Toolbox): if not toolbox._prepared: toolbox.prepare() toolbox._prepared = True tool_iter = toolbox.tools() else: tool_iter = toolbox.method_tools() methods = [] for tool in tool_iter: methods.append( { "name": tool.name, "description": tool.description, "arguments": tool.input_schema, "implementation": tool.implementation, } ) return methods toolbox_specs: dict[int, str] = {} if tool_defs: tools = {} gathered = _gather_tools(tool_defs, python_tools) # _gather_tools returns --functions tools first, then one per spec specs = [None] * (len(gathered) - len(tool_defs)) + list(tool_defs) for spec, tool in zip(specs, gathered): if hasattr(tool, "name"): tools[tool.name] = tool else: tools[tool.__class__.__name__] = tool if spec is not None and isinstance(tool, Toolbox): toolbox_specs[id(tool)] = spec else: tools = get_tools() if python_tools: for code_or_path in python_tools: for tool in _tools_from_code(code_or_path): tools[tool.name] = tool output_tools = [] output_toolboxes = [] tool_objects = [] toolbox_infos = [] for name, tool in sorted(tools.items()): if isinstance(tool, Tool): tool_objects.append(tool) output_tools.append( { "name": name, "description": tool.description, "arguments": tool.input_schema, "plugin": tool.plugin, } ) else: toolbox_class = tool if isinstance(tool, type) else tool.__class__ # Overriding tools() or prepare() means the toolbox generates # tools at runtime is_dynamic = any( getattr(toolbox_class, method) is not getattr(Toolbox, method) for method in ("tools", "prepare", "prepare_async") ) introspected = introspect_tools(tool) toolbox_infos.append((name, tool, toolbox_class, is_dynamic, introspected)) output_toolboxes.append( { "name": name, "dynamic": is_dynamic, "tools": [ { "name": tool_info["name"], "description": tool_info["description"], "arguments": tool_info["arguments"], } for tool_info in introspected ], } ) if json_: output = {"tools": output_tools, "toolboxes": output_toolboxes} if model is not None: output["server_side_tools"] = server_side_tools click.echo(json.dumps(output, indent=2)) else: for tool in tool_objects: sig = "()" if tool.implementation: sig = str(inspect.signature(tool.implementation)) click.echo( "{}{}{}\n".format( tool.name, sig, f" (plugin: {tool.plugin})" if tool.plugin else "", ) ) if tool.description: click.echo(textwrap.indent(tool.description.strip(), " ") + "\n") for name, toolbox, toolbox_class, is_dynamic, introspected in toolbox_infos: if is_dynamic and isinstance(toolbox, type): # A dynamic toolbox class has no tools until it is # instantiated - show its constructor and docstring instead try: constructor_sig = str(inspect.signature(toolbox_class)) except (ValueError, TypeError): constructor_sig = "(...)" plugin = getattr(toolbox_class, "plugin", None) click.echo( "{}{}{}\n".format( name, constructor_sig, f" (plugin: {plugin})" if plugin else "", ) ) doc = toolbox_class.__doc__ if doc: click.echo(textwrap.indent(inspect.cleandoc(doc), " ") + "\n") else: click.echo(toolbox_specs.get(id(toolbox), name) + ":\n") for tool_info in introspected: sig = "()" if tool_info["implementation"]: sig = ( str(inspect.signature(tool_info["implementation"])) .replace("(self, ", "(") .replace("(self)", "()") ) click.echo(f" {tool_info['name']}{sig}\n") if tool_info["description"]: click.echo( textwrap.indent(tool_info["description"].strip(), " ") + "\n" ) if model is not None: if server_side_tools: click.echo( f"Server-side tools for {model.model_id} " "(executed by the provider):\n" ) for tool_info in server_side_tools: click.echo(f"{tool_info['name']}{tool_info['signature']}\n") if tool_info["description"]: click.echo( textwrap.indent(tool_info["description"], " ") + "\n" ) else: click.echo(f"No server-side tools for {model.model_id}.") @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def aliases(): "Manage model aliases" @aliases.command(name="list") @click.option("json_", "--json", is_flag=True, help="Output as JSON") def aliases_list(json_): "List current aliases" to_output = [] for alias, model in get_model_aliases().items(): if alias != model.model_id: to_output.append((alias, model.model_id, "")) for alias, embedding_model in get_embedding_model_aliases().items(): if alias != embedding_model.model_id: to_output.append((alias, embedding_model.model_id, "embedding")) if json_: click.echo( json.dumps({key: value for key, value, type_ in to_output}, indent=4) ) return max_alias_length = max(len(a) for a, _, _ in to_output) fmt = "{alias:<" + str(max_alias_length) + "} : {model_id}{type_}" for alias, model_id, type_ in to_output: click.echo( fmt.format( alias=alias, model_id=model_id, type_=f" ({type_})" if type_ else "" ) ) @aliases.command(name="set") @click.argument("alias") @click.argument("model_id", required=False) @click.option( "-q", "--query", multiple=True, help="Set alias for model matching these strings", ) def aliases_set(alias, model_id, query): """ Set an alias for a model Example usage: \b llm aliases set luna gpt-5.6-luna Alternatively you can omit the model ID and specify one or more -q options. The first model matching all of those query strings will be used. \b llm aliases set luna -q gpt -q luna """ if not model_id: if not query: raise click.ClickException( "You must provide a model_id or at least one -q option" ) # Search for the first model matching all query strings found = None for model_with_aliases in get_models_with_aliases(): if all(model_with_aliases.matches(q) for q in query): found = model_with_aliases break if not found: raise click.ClickException( "No model found matching query: " + ", ".join(query) ) model_id = found.model.model_id set_alias(alias, model_id) click.echo( f"Alias '{alias}' set to model '{model_id}'", err=True, ) else: set_alias(alias, model_id) @aliases.command(name="remove") @click.argument("alias") def aliases_remove(alias): """ Remove an alias Example usage: \b $ llm aliases remove turbo """ try: remove_alias(alias) except KeyError as ex: raise click.ClickException(ex.args[0]) @aliases.command(name="path") def aliases_path(): "Output the path to the aliases.json file" click.echo(user_dir() / "aliases.json") @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def fragments(): """ Manage fragments that are stored in the database Fragments are reusable snippets of text that are shared across multiple prompts. """ @fragments.command(name="list") @click.option( "queries", "-q", "--query", multiple=True, help="Search for fragments matching these strings", ) @click.option("--aliases", is_flag=True, help="Show only fragments with aliases") @click.option("json_", "--json", is_flag=True, help="Output as JSON") def fragments_list(queries, aliases, json_): "List current fragments" db = sqlite_utils.Database(logs_db_path()) migrate(db) params = {} where_bits = [] if aliases: where_bits.append("fragment_aliases.alias is not null") for param_count, q in enumerate(queries, start=1): p = f"p{param_count}" params[p] = q where_bits.append(f""" (fragments.hash = :{p} or fragment_aliases.alias = :{p} or fragments.source like '%' || :{p} || '%' or fragments.content like '%' || :{p} || '%') """) where = "\n and\n ".join(where_bits) if where: where = " where " + where sql = f""" select fragments.hash, json_group_array(fragment_aliases.alias) filter ( where fragment_aliases.alias is not null ) as aliases, fragments.datetime_utc, fragments.source, fragments.content from fragments left join fragment_aliases on fragment_aliases.fragment_id = fragments.id {where} group by fragments.id, fragments.hash, fragments.content, fragments.datetime_utc, fragments.source order by fragments.datetime_utc """ results = list(db.query(sql, params)) for result in results: result["aliases"] = json.loads(result["aliases"]) if json_: click.echo(json.dumps(results, indent=4)) else: yaml.add_representer( str, lambda dumper, data: dumper.represent_scalar( "tag:yaml.org,2002:str", data, style="|" if "\n" in data else None ), ) for result in results: result["content"] = truncate_string(result["content"]) click.echo(yaml.dump([result], sort_keys=False, width=sys.maxsize).strip()) @fragments.command(name="set") @click.argument("alias", callback=validate_fragment_alias) @click.argument("fragment") def fragments_set(alias, fragment): """ Set an alias for a fragment Accepts an alias and a file path, URL, hash or '-' for stdin Example usage: \b llm fragments set mydocs ./docs.md """ db = sqlite_utils.Database(logs_db_path()) migrate(db) try: resolved = resolve_fragments(db, [fragment])[0] except FragmentNotFound as ex: raise click.ClickException(str(ex)) migrate(db) alias_sql = """ insert into fragment_aliases (alias, fragment_id) values (:alias, :fragment_id) on conflict(alias) do update set fragment_id = excluded.fragment_id; """ with db.atomic(): fragment_id = ensure_fragment(db, resolved) db.execute(alias_sql, {"alias": alias, "fragment_id": fragment_id}) @fragments.command(name="show") @click.argument("alias_or_hash") def fragments_show(alias_or_hash): """ Display the fragment stored under an alias or hash \b llm fragments show mydocs """ db = sqlite_utils.Database(logs_db_path()) migrate(db) try: resolved = resolve_fragments(db, [alias_or_hash])[0] except FragmentNotFound as ex: raise click.ClickException(str(ex)) click.echo(resolved) @fragments.command(name="remove") @click.argument("alias", callback=validate_fragment_alias) def fragments_remove(alias): """ Remove a fragment alias Example usage: \b llm fragments remove docs """ db = sqlite_utils.Database(logs_db_path()) migrate(db) db.execute("delete from fragment_aliases where alias = :alias", {"alias": alias}) @fragments.command(name="loaders") def fragments_loaders(): """Show fragment loaders registered by plugins""" from llm import get_fragment_loaders found = False for prefix, loader in get_fragment_loaders().items(): if found: # Extra newline on all after the first click.echo("") found = True docs = "Undocumented" if loader.__doc__: docs = textwrap.dedent(loader.__doc__).strip() click.echo(f"{prefix}:") click.echo(textwrap.indent(docs, " ")) if not found: click.echo("No fragment loaders found") @cli.command(name="plugins") @click.option("--all", help="Include built-in default plugins", is_flag=True) @click.option( "hooks", "--hook", help="Filter for plugins that implement this hook", multiple=True ) def plugins_list(all, hooks): "List installed plugins" plugins = get_plugins(all) hooks = set(hooks) if hooks: plugins = [plugin for plugin in plugins if hooks.intersection(plugin["hooks"])] click.echo(json.dumps(plugins, indent=2)) def display_truncated(text): console_width = shutil.get_terminal_size()[0] if len(text) > console_width: return text[: console_width - 3] + "..." else: return text @cli.command() @click.argument("packages", nargs=-1, required=False) @click.option( "-U", "--upgrade", is_flag=True, help="Upgrade packages to latest version" ) @click.option( "-e", "--editable", help="Install a project in editable mode from this path", ) @click.option( "--force-reinstall", is_flag=True, help="Reinstall all packages even if they are already up-to-date", ) @click.option( "--no-cache-dir", is_flag=True, help="Disable the cache", ) @click.option( "--pre", is_flag=True, help="Include pre-release and development versions", ) def install(packages, upgrade, editable, force_reinstall, no_cache_dir, pre): """Install packages from PyPI into the same environment as LLM""" args = ["pip", "install"] if upgrade: args += ["--upgrade"] if editable: args += ["--editable", editable] if force_reinstall: args += ["--force-reinstall"] if no_cache_dir: args += ["--no-cache-dir"] if pre: args += ["--pre"] args += list(packages) sys.argv = args run_module("pip", run_name="__main__") @cli.command() @click.argument("packages", nargs=-1, required=True) @click.option("-y", "--yes", is_flag=True, help="Don't ask for confirmation") def uninstall(packages, yes): """Uninstall Python packages from the LLM environment""" sys.argv = ["pip", "uninstall"] + list(packages) + (["-y"] if yes else []) run_module("pip", run_name="__main__") @cli.command() @click.argument("collection", required=False) @click.argument("id", required=False) @click.option( "-i", "--input", type=click.Path(exists=True, readable=True, allow_dash=True), help="File to embed", ) @click.option( "-m", "--model", help="Embedding model to use", envvar="LLM_EMBEDDING_MODEL" ) @click.option("--store", is_flag=True, help="Store the text itself in the database") @click.option( "-d", "--database", type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), envvar="LLM_EMBEDDINGS_DB", ) @click.option( "-c", "--content", help="Content to embed", ) @click.option("--binary", is_flag=True, help="Treat input as binary data") @click.option( "--metadata", help="JSON object metadata to store", callback=json_validator("metadata"), ) @click.option( "format_", "-f", "--format", type=click.Choice(["json", "blob", "base64", "hex"]), help="Output format", ) def embed( collection, id, input, model, store, database, content, binary, metadata, format_ ): """Embed text and store or return the result""" if collection and not id: raise click.ClickException("Must provide both collection and id") if store and not collection: raise click.ClickException("Must provide collection when using --store") # Lazy load this because we do not need it for -c or -i versions def get_db(): if database: return sqlite_utils.Database(database) else: return sqlite_utils.Database(user_dir() / "embeddings.db") collection_obj = None model_obj = None if collection: db = get_db() if Collection.exists(db, collection): # Load existing collection and use its model collection_obj = Collection(collection, db) model_obj = collection_obj.model() else: # We will create a new one, but that means model is required if not model: model = get_default_embedding_model() if model is None: raise click.ClickException( "You need to specify an embedding model (no default model is set)" ) collection_obj = Collection(collection, db=db, model_id=model) model_obj = collection_obj.model() if model_obj is None: if model is None: model = get_default_embedding_model() try: model_obj = get_embedding_model(model) except UnknownModelError: raise click.ClickException( "You need to specify an embedding model (no default model is set)" ) show_output = True if collection and (format_ is None): show_output = False # Resolve input text if not content: if not input or input == "-": # Read from stdin input_source = sys.stdin.buffer if binary else sys.stdin content = input_source.read() else: mode = "rb" if binary else "r" with open(input, mode) as f: content = f.read() if not content: raise click.ClickException("No content provided") if collection_obj: embedding = collection_obj.embed(id, content, metadata=metadata, store=store) else: embedding = model_obj.embed(content) if show_output: if format_ == "json" or format_ is None: click.echo(json.dumps(embedding)) elif format_ == "blob": click.echo(encode(embedding)) elif format_ == "base64": click.echo(base64.b64encode(encode(embedding)).decode("ascii")) elif format_ == "hex": click.echo(encode(embedding).hex()) @cli.command() @click.argument("collection") @click.argument( "input_path", type=click.Path(exists=True, dir_okay=False, allow_dash=True, readable=True), required=False, ) @click.option( "--format", type=click.Choice(["json", "csv", "tsv", "nl"]), help="Format of input file - defaults to auto-detect", ) @click.option( "--files", type=(click.Path(file_okay=False, dir_okay=True, allow_dash=False), str), multiple=True, help="Embed files in this directory - specify directory and glob pattern", ) @click.option( "encodings", "--encoding", help="Encodings to try when reading --files", multiple=True, ) @click.option("--binary", is_flag=True, help="Treat --files as binary data") @click.option("--sql", help="Read input using this SQL query") @click.option( "--attach", type=(str, click.Path(file_okay=True, dir_okay=False, allow_dash=False)), multiple=True, help="Additional databases to attach - specify alias and file path", ) @click.option( "--batch-size", type=int, help="Batch size to use when running embeddings" ) @click.option("--prefix", help="Prefix to add to the IDs", default="") @click.option( "-m", "--model", help="Embedding model to use", envvar="LLM_EMBEDDING_MODEL" ) @click.option( "--prepend", help="Prepend this string to all content before embedding", ) @click.option("--store", is_flag=True, help="Store the text itself in the database") @click.option( "-d", "--database", type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), envvar="LLM_EMBEDDINGS_DB", ) def embed_multi( collection, input_path, format, files, encodings, binary, sql, attach, batch_size, prefix, model, prepend, store, database, ): """ Store embeddings for multiple strings at once in the specified collection. Input data can come from one of three sources: \b 1. A CSV, TSV, JSON or JSONL file: - CSV/TSV: First column is ID, remaining columns concatenated as content - JSON: Array of objects with "id" field and content fields - JSONL: Newline-delimited JSON objects \b Examples: llm embed-multi docs input.csv cat data.json | llm embed-multi docs - llm embed-multi docs input.json --format json \b 2. A SQL query against a SQLite database: - First column returned is used as ID - Other columns concatenated to form content \b Examples: llm embed-multi docs --sql "SELECT id, title, body FROM posts" llm embed-multi docs --attach blog blog.db --sql "SELECT id, content FROM blog.posts" \b 3. Files in directories matching glob patterns: - Each file becomes one embedding - Relative file paths become IDs \b Examples: llm embed-multi docs --files docs '**/*.md' llm embed-multi images --files photos '*.jpg' --binary llm embed-multi texts --files texts '*.txt' --encoding utf-8 --encoding latin-1 """ if binary and not files: raise click.UsageError("--binary must be used with --files") if binary and encodings: raise click.UsageError("--binary cannot be used with --encoding") if not input_path and not sql and not files: raise click.UsageError("Either --sql or input path or --files is required") if files and (input_path or sql or format): raise click.UsageError("Cannot use --files with --sql, input path or --format") if database: db = sqlite_utils.Database(database) else: db = sqlite_utils.Database(user_dir() / "embeddings.db") for alias, attach_path in attach: db.attach(alias, attach_path) try: collection_obj = Collection( collection, db=db, model_id=model or get_default_embedding_model() ) except ValueError: raise click.ClickException( "You need to specify an embedding model (no default model is set)" ) expected_length = None if files: encodings = encodings or ("utf-8", "latin-1") def count_files(): i = 0 for directory, pattern in files: for path in pathlib.Path(directory).glob(pattern): i += 1 return i def iterate_files(): for directory, pattern in files: p = pathlib.Path(directory) if not p.exists() or not p.is_dir(): # fixes issue/274 - raise error if directory does not exist raise click.UsageError(f"Invalid directory: {directory}") for path in pathlib.Path(directory).glob(pattern): if path.is_dir(): continue # fixed issue/280 - skip directories relative = path.relative_to(directory) content = None if binary: content = path.read_bytes() else: for encoding in encodings: try: content = path.read_text(encoding=encoding) except UnicodeDecodeError: continue if content is None: # Log to stderr click.echo( f"Could not decode text in file {path}", err=True, ) else: yield {"id": str(relative), "content": content} expected_length = count_files() rows = iterate_files() elif sql: rows = db.query(sql) count_sql = f"select count(*) as c from ({sql})" expected_length = next(db.query(count_sql))["c"] else: def load_rows(fp): return rows_from_file(fp, Format[format.upper()] if format else None)[0] try: if input_path != "-": # Read the file twice - first time is to get a count expected_length = 0 with open(input_path, "rb") as fp: for _ in load_rows(fp): expected_length += 1 if input_path != "-": def rows_from_input(): with open(input_path, "rb") as fp: yield from load_rows(fp) rows = rows_from_input() else: rows = load_rows(io.BufferedReader(sys.stdin.buffer)) except json.JSONDecodeError as ex: raise click.ClickException(str(ex)) with click.progressbar( rows, label="Embedding", show_percent=True, length=expected_length ) as rows: def tuples() -> Iterable[tuple[str, bytes | str]]: for row in rows: values = list(row.values()) id: str = prefix + str(values[0]) content: bytes | str | None = None if binary: content = cast(bytes, values[1]) else: content = " ".join(v or "" for v in values[1:]) if prepend and isinstance(content, str): content = prepend + content yield id, content or "" embed_kwargs = {"store": store} if batch_size: embed_kwargs["batch_size"] = batch_size collection_obj.embed_multi(tuples(), **embed_kwargs) @cli.command() @click.argument("collection") @click.argument("id", required=False) @click.option( "-i", "--input", type=click.Path(exists=True, readable=True, allow_dash=True), help="File to embed for comparison", ) @click.option("-c", "--content", help="Content to embed for comparison") @click.option("--binary", is_flag=True, help="Treat input as binary data") @click.option( "-n", "--number", type=int, default=10, help="Number of results to return" ) @click.option("-p", "--plain", is_flag=True, help="Output in plain text format") @click.option( "-d", "--database", type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), envvar="LLM_EMBEDDINGS_DB", ) @click.option("--prefix", help="Just IDs with this prefix", default="") def similar(collection, id, input, content, binary, number, plain, database, prefix): """ Return top N similar IDs from a collection using cosine similarity. Example usage: \b llm similar my-collection -c "I like cats" Or to find content similar to a specific stored ID: \b llm similar my-collection 1234 """ if not id and not content and not input: raise click.ClickException("Must provide content or an ID for the comparison") if database: db = sqlite_utils.Database(database) else: db = sqlite_utils.Database(user_dir() / "embeddings.db") if not db["embeddings"].exists(): raise click.ClickException("No embeddings table found in database") try: collection_obj = Collection(collection, db, create=False) except Collection.DoesNotExist: raise click.ClickException("Collection does not exist") if id: try: results = collection_obj.similar_by_id(id, number, prefix=prefix) except Collection.DoesNotExist: raise click.ClickException("ID not found in collection") else: # Resolve input text if not content: if not input or input == "-": # Read from stdin input_source = sys.stdin.buffer if binary else sys.stdin content = input_source.read() else: mode = "rb" if binary else "r" with open(input, mode) as f: content = f.read() if not content: raise click.ClickException("No content provided") results = collection_obj.similar(content, number, prefix=prefix) for result in results: if plain: click.echo(f"{result.id} ({result.score})\n") if result.content: click.echo(textwrap.indent(result.content, " ")) if result.metadata: click.echo(textwrap.indent(json.dumps(result.metadata), " ")) click.echo("") else: click.echo(json.dumps(asdict(result))) @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def embed_models(): "Manage available embedding models" @embed_models.command(name="list") @click.option( "-q", "--query", multiple=True, help="Search for embedding models matching these strings", ) def embed_models_list(query): "List available embedding models" output = [] for model_with_aliases in get_embedding_models_with_aliases(): if query and not all(model_with_aliases.matches(q) for q in query): continue s = str(model_with_aliases.model) if model_with_aliases.aliases: s += " (aliases: {})".format(", ".join(model_with_aliases.aliases)) output.append(s) click.echo("\n".join(output)) @embed_models.command(name="default") @click.argument("model", required=False) @click.option( "--remove-default", is_flag=True, help="Reset to specifying no default model" ) def embed_models_default(model, remove_default): "Show or set the default embedding model" if not model and not remove_default: default = get_default_embedding_model() if default is None: click.echo("", err=True) else: click.echo(default) return # Validate it is a known model try: if remove_default: set_default_embedding_model(None) else: model = get_embedding_model(model) set_default_embedding_model(model.model_id) except KeyError: raise click.ClickException(f"Unknown embedding model: {model}") @cli.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def collections(): "View and manage collections of embeddings" @collections.command(name="path") def collections_path(): "Output the path to the embeddings database" click.echo(user_dir() / "embeddings.db") @collections.command(name="list") @click.option( "-d", "--database", type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), envvar="LLM_EMBEDDINGS_DB", help="Path to embeddings database", ) @click.option("json_", "--json", is_flag=True, help="Output as JSON") def embed_db_collections(database, json_): "View a list of collections" database = database or (user_dir() / "embeddings.db") db = sqlite_utils.Database(str(database)) if not db["collections"].exists(): raise click.ClickException(f"No collections table found in {database}") rows = db.query(""" select collections.name, collections.model, count(embeddings.id) as num_embeddings from collections left join embeddings on collections.id = embeddings.collection_id group by collections.name, collections.model """) if json_: click.echo(json.dumps(list(rows), indent=4)) else: for row in rows: click.echo("{}: {}".format(row["name"], row["model"])) click.echo( " {} embedding{}".format( row["num_embeddings"], "s" if row["num_embeddings"] != 1 else "" ) ) @collections.command(name="delete") @click.argument("collection") @click.option( "-d", "--database", type=click.Path(file_okay=True, allow_dash=False, dir_okay=False, writable=True), envvar="LLM_EMBEDDINGS_DB", help="Path to embeddings database", ) def collections_delete(collection, database): """ Delete the specified collection Example usage: \b llm collections delete my-collection """ database = database or (user_dir() / "embeddings.db") db = sqlite_utils.Database(str(database)) try: collection_obj = Collection(collection, db, create=False) except Collection.DoesNotExist: raise click.ClickException("Collection does not exist") collection_obj.delete() @models.group( cls=DefaultGroup, default="list", default_if_no_args=True, ) def options(): "Manage default options for models" @options.command(name="list") def options_list(): """ List default options for all models Example usage: \b llm models options list """ options = get_all_model_options() if not options: click.echo("No default options set for any models.", err=True) return for model_id, model_options in options.items(): click.echo(f"{model_id}:") for key, value in model_options.items(): click.echo(f" {key}: {value}") @options.command(name="show") @click.argument("model") def options_show(model): """ List default options set for a specific model Example usage: \b llm models options show gpt-4.1 """ import llm try: # Resolve alias to model ID model_obj = llm.get_model(model) model_id = model_obj.model_id except llm.UnknownModelError: # Use as-is if not found model_id = model options = get_model_options(model_id) if not options: click.echo(f"No default options set for model '{model_id}'.", err=True) return for key, value in options.items(): click.echo(f"{key}: {value}") @options.command(name="set") @click.argument("model") @click.argument("key") @click.argument("value") def options_set(model, key, value): """ Set a default option for a model Example usage: \b llm models options set gpt-4.1 temperature 0.5 """ import llm try: # Resolve alias to model ID model_obj = llm.get_model(model) model_id = model_obj.model_id # Validate option against model schema try: # Create a test Options object to validate test_options = {key: value} model_obj.Options(**test_options) except pydantic.ValidationError as ex: raise click.ClickException(render_errors(ex.errors())) except llm.UnknownModelError: # Use as-is if not found model_id = model set_model_option(model_id, key, value) click.echo(f"Set default option {key}={value} for model {model_id}", err=True) @options.command(name="clear") @click.argument("model") @click.argument("key", required=False) def options_clear(model, key): """ Clear default option(s) for a model Example usage: \b llm models options clear gpt-4.1 # Or for a single option llm models options clear gpt-4.1 temperature """ import llm try: # Resolve alias to model ID model_obj = llm.get_model(model) model_id = model_obj.model_id except llm.UnknownModelError: # Use as-is if not found model_id = model cleared_keys = [] if not key: cleared_keys = list(get_model_options(model_id).keys()) for key_ in cleared_keys: clear_model_option(model_id, key_) else: cleared_keys.append(key) clear_model_option(model_id, key) if cleared_keys: if len(cleared_keys) == 1: click.echo(f"Cleared option '{cleared_keys[0]}' for model {model_id}") else: click.echo( f"Cleared {', '.join(cleared_keys)} options for model {model_id}" ) def template_dir(): path = user_dir() / "templates" path.mkdir(parents=True, exist_ok=True) return path def logs_db_path(): return user_dir() / "logs.db" def get_history(chat_id): if chat_id is None: return None, [] log_path = logs_db_path() db = sqlite_utils.Database(log_path) migrate(db) if chat_id == -1: # Return the most recent chat last_row = list(db["logs"].rows_where(order_by="-id", limit=1)) if last_row: chat_id = last_row[0].get("chat_id") or last_row[0].get("id") else: # Database is empty return None, [] rows = db["logs"].rows_where( "id = ? or chat_id = ?", [chat_id, chat_id], order_by="id" ) return chat_id, rows def render_errors(errors): output = [] for error in errors: output.append(", ".join(error["loc"])) output.append(" " + error["msg"]) return "\n".join(output) load_plugins() pm.hook.register_commands(cli=cli) def _human_readable_size(size_bytes): if size_bytes == 0: return "0B" size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") i = 0 while size_bytes >= 1024 and i < len(size_name) - 1: size_bytes /= 1024.0 i += 1 return f"{size_bytes:.2f}{size_name[i]}" def logs_on(): return not (user_dir() / "logs-off").exists() def get_all_model_options() -> dict: """ Get all default options for all models """ path = user_dir() / "model_options.json" if not path.exists(): return {} try: options = json.loads(path.read_text()) except json.JSONDecodeError: return {} return options def get_model_options(model_id: str) -> dict: """ Get default options for a specific model Args: model_id: Return options for model with this ID Returns: A dictionary of model options """ path = user_dir() / "model_options.json" if not path.exists(): return {} try: options = json.loads(path.read_text()) except json.JSONDecodeError: return {} return options.get(model_id, {}) def set_model_option(model_id: str, key: str, value: Any) -> None: """ Set a default option for a model. Args: model_id: The model ID key: The option key value: The option value """ path = user_dir() / "model_options.json" if path.exists(): try: options = json.loads(path.read_text()) except json.JSONDecodeError: options = {} else: options = {} # Ensure the model has an entry if model_id not in options: options[model_id] = {} # Set the option options[model_id][key] = value # Save the options path.write_text(json.dumps(options, indent=2)) def clear_model_option(model_id: str, key: str) -> None: """ Clear a model option Args: model_id: The model ID key: Key to clear """ path = user_dir() / "model_options.json" if not path.exists(): return try: options = json.loads(path.read_text()) except json.JSONDecodeError: return if model_id not in options: return if key in options[model_id]: del options[model_id][key] if not options[model_id]: del options[model_id] path.write_text(json.dumps(options, indent=2)) class LoadTemplateError(ValueError): pass def _parse_yaml_template(name, content): try: loaded = yaml.safe_load(content) except yaml.YAMLError as ex: raise LoadTemplateError(f"Invalid YAML: {ex!s}") if isinstance(loaded, str): return Template(name=name, prompt=loaded) loaded["name"] = name try: return Template(**loaded) except pydantic.ValidationError as ex: msg = "A validation error occurred:\n" msg += render_errors(ex.errors()) raise LoadTemplateError(msg) def load_template(name: str) -> Template: "Load template, or raise LoadTemplateError(msg)" if name.startswith(("https://", "http://")): response = httpx.get(name) try: response.raise_for_status() except httpx.HTTPStatusError as ex: raise LoadTemplateError(f"Could not load template {name}: {ex}") return _parse_yaml_template(name, response.text) potential_path = pathlib.Path(name) if has_plugin_prefix(name) and not potential_path.exists(): prefix, rest = name.split(":", 1) loaders = get_template_loaders() if prefix not in loaders: raise LoadTemplateError(f"Unknown template prefix: {prefix}") loader = loaders[prefix] try: return loader(rest) except Exception as ex: # noqa: BLE001 raise LoadTemplateError(f"Could not load template {name}: {ex}") # Try local file if potential_path.exists(): path = potential_path else: # Look for template in template_dir() path = template_dir() / f"{name}.yaml" if not path.exists(): raise LoadTemplateError(f"Invalid template: {name}") content = path.read_text() template_obj = _parse_yaml_template(name, content) # We trust functions here because they came from the filesystem template_obj._functions_is_trusted = True return template_obj def _tools_from_code(code_or_path: str) -> list[Tool]: """ Treat all Python functions in the code as tools """ if "\n" not in code_or_path and code_or_path.endswith(".py"): try: code_or_path = pathlib.Path(code_or_path).read_text() except FileNotFoundError: raise click.ClickException(f"File not found: {code_or_path}") namespace: dict[str, Any] = {} tools = [] try: exec(code_or_path, namespace) # noqa: S102 except SyntaxError as ex: raise click.ClickException(f"Error in --functions definition: {ex}") # Register all callables in the locals dict: for name, value in namespace.items(): if callable(value) and not name.startswith("_"): tools.append(Tool.function(value)) return tools def _debug_tool_call(_, tool_call, tool_result): click.echo( click.style( f"\nTool call: {tool_call.name}({tool_call.arguments})", fg="yellow", bold=True, ), err=True, ) output = "" attachments = "" if tool_result.attachments: attachments += "\nAttachments:\n" for attachment in tool_result.attachments: attachments += f" {attachment!r}\n" try: output = json.dumps(json.loads(tool_result.output), indent=2) except ValueError: output = tool_result.output output += attachments click.echo( click.style( textwrap.indent(output, " ") + ("\n" if not tool_result.exception else ""), fg="green", bold=True, ), err=True, ) if tool_result.exception: click.echo( click.style( f" Exception: {tool_result.exception}", fg="red", bold=True, ), err=True, ) def _approve_tool_call(_, tool_call): click.echo( click.style( f"Tool call: {tool_call.name}({tool_call.arguments})", fg="yellow", bold=True, ), err=True, ) if not click.confirm("Approve tool call?"): raise CancelToolCall("User cancelled tool call") def _gather_tools( tool_specs: list[str], python_tools: list[str], model=None ) -> list[Tool | Toolbox | ServerSideTool]: tools: list[Tool | Toolbox | ServerSideTool] = [] if python_tools: for code_or_path in python_tools: tools.extend(_tools_from_code(code_or_path)) registered_tools = get_tools() server_side_tool_classes = { tool_class.__name__: tool_class for tool_class in ( model.supported_server_side_tools if model is not None else () ) } available_tools = {**registered_tools, **server_side_tool_classes} registered_classes = { key: value for key, value in available_tools.items() if inspect.isclass(value) } bad_tools = [ tool for tool in tool_specs if tool.split("(", 1)[0].strip() not in available_tools ] if bad_tools: raise click.ClickException( "Tool(s) {} not found. Available tools: {}".format( ", ".join(bad_tools), ", ".join(available_tools.keys()) ) ) for tool_spec in tool_specs: if not tool_spec[0].isupper(): # It's a function tools.append(available_tools[tool_spec]) else: # It's a class tools.append(instantiate_from_spec(registered_classes, tool_spec)) return tools def _tool_chain_kwargs( tool_specs, python_tools, tools_debug, tools_approve, chain_limit, model=None ): """Build Conversation.chain() keyword arguments for CLI-selected tools.""" tool_implementations = _gather_tools(tool_specs, python_tools, model=model) if not tool_implementations: return {} kwargs = { "tools": tool_implementations, "chain_limit": chain_limit, } if tools_debug: kwargs["after_call"] = _debug_tool_call if tools_approve: kwargs["before_call"] = _approve_tool_call return kwargs def _get_conversation_tools(conversation, tools): if not conversation or tools: return None if conversation.responses: # Copy plugin tools from first response in conversation initial_tools = conversation.responses[0].prompt.tools if initial_tools: # Only tools from plugins: return [tool.name for tool in initial_tools if tool.plugin] elif conversation.loaded_tools: # Conversation loaded from the message store - tool names and # toolbox specs were read from turn_tools instead of rebuilt # responses. return list(conversation.loaded_tools) ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1785864218.5841985 llm-0.32/llm/default_plugins/0000755000175100017510000000000015234420033015635 5ustar00runnerrunner././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/default_plugins/__init__.py0000644000175100017510000000000015234420023017733 0ustar00runnerrunner././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/default_plugins/default_tools.py0000644000175100017510000000022615234420023021052 0ustar00runnerrunnerimport llm from llm.tools import llm_time, llm_version @llm.hookimpl def register_tools(register): register(llm_version) register(llm_time) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/default_plugins/openai_models.py0000644000175100017510000033230215234420023021027 0ustar00runnerrunnerimport datetime import json import os import sys from collections.abc import AsyncGenerator, Iterable, Iterator from enum import Enum from typing import Any, ClassVar, Literal import click import httpx import openai import sqlite_utils import yaml from pydantic import Field, ValidationError, create_model, field_validator import llm from llm import ( AsyncConversation, AsyncKeyModel, AsyncResponse, Conversation, EmbeddingModel, KeyModel, Prompt, Response, hookimpl, ) from llm.models import _partition_tools from llm.parts import StreamEvent from llm.utils import ( dicts_to_table_string, logging_client, remove_dict_none_values, simplify_usage_dict, ) @hookimpl def register_models(register): # GPT-4o register( Chat( "gpt-4o", vision=True, service_tier=True, supports_schema=True, supports_tools=True, ), AsyncChat( "gpt-4o", vision=True, service_tier=True, supports_schema=True, supports_tools=True, ), aliases=("4o",), ) register( Chat( "gpt-4o-mini", vision=True, service_tier=True, supports_schema=True, supports_tools=True, ), AsyncChat( "gpt-4o-mini", vision=True, service_tier=True, supports_schema=True, supports_tools=True, ), aliases=("4o-mini",), ) # GPT-4.1 for model_id in ("gpt-4.1", "gpt-4.1-mini", "gpt-4.1-nano"): register( Chat( model_id, vision=True, service_tier=True, supports_schema=True, supports_tools=True, ), AsyncChat( model_id, vision=True, service_tier=True, supports_schema=True, supports_tools=True, ), aliases=(model_id.replace("gpt-", ""),), ) # 3.5 and 4 register( Chat("gpt-3.5-turbo", service_tier=True), AsyncChat("gpt-3.5-turbo", service_tier=True), aliases=("3.5", "chatgpt"), ) register( Chat("gpt-3.5-turbo-16k", service_tier=True), AsyncChat("gpt-3.5-turbo-16k", service_tier=True), aliases=("chatgpt-16k", "3.5-16k"), ) register( Chat("gpt-4", service_tier=True), AsyncChat("gpt-4", service_tier=True), aliases=("4", "gpt4"), ) # GPT-4 Turbo models register( Chat("gpt-4-turbo-2024-04-09", service_tier=True), AsyncChat("gpt-4-turbo-2024-04-09", service_tier=True), ) register( Chat("gpt-4-turbo", service_tier=True), AsyncChat("gpt-4-turbo", service_tier=True), aliases=("gpt-4-turbo-preview", "4-turbo", "4t"), ) # o1 for model_id in ("o1", "o1-2024-12-17"): register( Responses( model_id, vision=True, can_stream=False, reasoning=True, service_tier=True, supports_schema=True, supports_tools=True, ), AsyncResponses( model_id, vision=True, can_stream=False, reasoning=True, service_tier=True, supports_schema=True, supports_tools=True, ), ) register( Responses( "o3-mini", reasoning=True, service_tier=True, supports_schema=True, supports_tools=True, ), AsyncResponses( "o3-mini", reasoning=True, service_tier=True, supports_schema=True, supports_tools=True, ), ) register( Responses( "o3", vision=True, reasoning=True, service_tier=True, supports_schema=True, supports_tools=True, ), AsyncResponses( "o3", vision=True, reasoning=True, service_tier=True, supports_schema=True, supports_tools=True, ), ) register( Responses( "o4-mini", vision=True, reasoning=True, service_tier=True, supports_schema=True, supports_tools=True, ), AsyncResponses( "o4-mini", vision=True, reasoning=True, service_tier=True, supports_schema=True, supports_tools=True, ), ) # GPT-5 for model_id in ( "gpt-5", "gpt-5-mini", "gpt-5-nano", "gpt-5-2025-08-07", "gpt-5-mini-2025-08-07", "gpt-5-nano-2025-08-07", ): register( Responses( model_id, vision=True, reasoning=True, verbosity=True, service_tier=True, supports_schema=True, supports_tools=True, ), AsyncResponses( model_id, vision=True, reasoning=True, verbosity=True, service_tier=True, supports_schema=True, supports_tools=True, ), ) # GPT-5.1 register( Responses( "gpt-5.1", vision=True, reasoning=True, verbosity=True, service_tier=True, supports_schema=True, supports_tools=True, ), AsyncResponses( "gpt-5.1", vision=True, reasoning=True, verbosity=True, service_tier=True, supports_schema=True, supports_tools=True, ), ) # GPT-5.2 for model_id in ("gpt-5.2", "gpt-5.2-chat-latest"): register( Responses( model_id, vision=True, reasoning=True, verbosity=True, service_tier=True, supports_schema=True, supports_tools=True, ), AsyncResponses( model_id, vision=True, reasoning=True, verbosity=True, service_tier=True, supports_schema=True, supports_tools=True, ), ) # "gpt-5.2-pro" is Responses API only # GPT-5.4 for model_id in ( "gpt-5.4", "gpt-5.4-2026-03-05", "gpt-5.4-mini", "gpt-5.4-mini-2026-03-17", "gpt-5.4-nano", "gpt-5.4-nano-2026-03-17", ): register( Responses( model_id, vision=True, reasoning=True, verbosity=True, image_detail_original=True, service_tier=True, supports_schema=True, supports_tools=True, ), AsyncResponses( model_id, vision=True, reasoning=True, verbosity=True, image_detail_original=True, service_tier=True, supports_schema=True, supports_tools=True, ), ) # GPT-5.5 — routes through the Responses API by default; pass # ``-o chat_completions 1`` to fall back to /v1/chat/completions. for model_id in ( "gpt-5.5", "gpt-5.5-2026-04-23", ): register( Responses( model_id, vision=True, reasoning=True, verbosity=True, image_detail_original=True, service_tier=True, supports_schema=True, supports_tools=True, ), AsyncResponses( model_id, vision=True, reasoning=True, verbosity=True, image_detail_original=True, service_tier=True, supports_schema=True, supports_tools=True, ), ) # GPT-5.6 for model_id in ("gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"): register( Responses( model_id, vision=True, reasoning=True, verbosity=True, image_detail_original=True, service_tier=True, supports_schema=True, supports_tools=True, ), AsyncResponses( model_id, vision=True, reasoning=True, verbosity=True, image_detail_original=True, service_tier=True, supports_schema=True, supports_tools=True, ), ) # The -instruct completion model register( Completion("gpt-3.5-turbo-instruct", default_max_tokens=256), aliases=("3.5-instruct", "chatgpt-instruct"), ) # Load extra models extra_path = llm.user_dir() / "extra-openai-models.yaml" if not extra_path.exists(): return with open(extra_path) as f: extra_models = yaml.safe_load(f) for extra_model in extra_models: model_id = extra_model["model_id"] aliases = extra_model.get("aliases", []) model_name = extra_model["model_name"] api_base = extra_model.get("api_base") api_type = extra_model.get("api_type") api_version = extra_model.get("api_version") api_engine = extra_model.get("api_engine") headers = extra_model.get("headers") reasoning = extra_model.get("reasoning") kwargs = {} if extra_model.get("can_stream") is False: kwargs["can_stream"] = False if extra_model.get("supports_schema") is True: kwargs["supports_schema"] = True if extra_model.get("supports_tools") is True: kwargs["supports_tools"] = True if extra_model.get("vision") is True: kwargs["vision"] = True if extra_model.get("audio") is True: kwargs["audio"] = True if extra_model.get("service_tier") is True: kwargs["service_tier"] = True if extra_model.get("completion"): klass = Completion async_klass = None elif extra_model.get("responses"): klass = Responses async_klass = AsyncResponses else: klass = Chat async_klass = AsyncChat model_kwargs = dict( model_id=model_id, model_name=model_name, api_base=api_base, api_type=api_type, api_version=api_version, api_engine=api_engine, headers=headers, reasoning=reasoning, **kwargs, ) chat_model = klass(**model_kwargs) async_model = async_klass(**model_kwargs) if async_klass else None if api_base: chat_model.needs_key = None if async_model: async_model.needs_key = None if extra_model.get("api_key_name"): chat_model.needs_key = extra_model["api_key_name"] if async_model: async_model.needs_key = extra_model["api_key_name"] register( chat_model, async_model, aliases=aliases, ) @hookimpl def register_embedding_models(register): register( OpenAIEmbeddingModel("text-embedding-ada-002", "text-embedding-ada-002"), aliases=( "ada", "ada-002", ), ) register( OpenAIEmbeddingModel("text-embedding-3-small", "text-embedding-3-small"), aliases=("3-small",), ) register( OpenAIEmbeddingModel("text-embedding-3-large", "text-embedding-3-large"), aliases=("3-large",), ) # With varying dimensions register( OpenAIEmbeddingModel( "text-embedding-3-small-512", "text-embedding-3-small", 512 ), aliases=("3-small-512",), ) register( OpenAIEmbeddingModel( "text-embedding-3-large-256", "text-embedding-3-large", 256 ), aliases=("3-large-256",), ) register( OpenAIEmbeddingModel( "text-embedding-3-large-1024", "text-embedding-3-large", 1024 ), aliases=("3-large-1024",), ) class OpenAIEmbeddingModel(EmbeddingModel): needs_key = "openai" key_env_var = "OPENAI_API_KEY" batch_size = 100 def __init__(self, model_id, openai_model_id, dimensions=None): self.model_id = model_id self.openai_model_id = openai_model_id self.dimensions = dimensions def embed_batch(self, items: Iterable[str | bytes]) -> Iterator[list[float]]: kwargs = { "input": items, "model": self.openai_model_id, } if self.dimensions: kwargs["dimensions"] = self.dimensions client = openai.OpenAI(api_key=self.get_key()) results = client.embeddings.create(**kwargs).data return ([float(r) for r in result.embedding] for result in results) @hookimpl def register_commands(cli): from llm.cli import ( AttachmentType, attachment_types_callback, schema_option, tool_options, ) @cli.group(name="openai") def openai_(): "Commands for working with OpenAI and OpenAI-compatible APIs" @openai_.command() @click.argument("url") @click.argument("prompt", required=False) @click.option( "model_id", "-m", "--model", help="Model ID (required unless --models or provided by template)", ) @click.option("-s", "--system", help="System prompt to use") @click.option("-t", "--template", help="Template to use") @click.option( "param", "-p", "--param", multiple=True, type=(str, str), help="Parameters for template", ) @click.option( "options", "-o", "--option", type=(str, str), multiple=True, help="key/value options for the model", ) @schema_option @click.option( "--schema-multi", help="JSON schema to use for multiple results", ) @click.option( "attachments", "-a", "--attachment", type=AttachmentType(), multiple=True, help="Attachment path or URL or -", ) @click.option( "attachment_types", "--at", "--attachment-type", type=(str, str), multiple=True, callback=attachment_types_callback, help="\b\nAttachment with explicit mimetype,\n--at image.jpg image/jpeg", ) @tool_options @click.option("--key", help="API key or stored key alias to send") @click.option( "headers", "-H", "--header", type=(str, str), multiple=True, help="Additional HTTP header", ) @click.option( "use_responses", "--responses", is_flag=True, help="Use the Responses API instead of Chat Completions", ) @click.option( "force_chat", "--chat", is_flag=True, help="Start an interactive chat", ) @click.option( "list_models", "--models", is_flag=True, help="List model IDs from the endpoint and exit", ) @click.option("--no-stream", is_flag=True, help="Do not stream output") @click.option("-R", "--hide-reasoning", is_flag=True, help="Hide reasoning output") def endpoint( url, prompt, model_id, system, template, param, options, schema_input, schema_multi, attachments, attachment_types, tools, python_tools, tools_debug, tools_approve, chain_limit, key, headers, use_responses, force_chat, list_models, no_stream, hide_reasoning, ): """ Run against an OpenAI-compatible endpoint without logging. PROMPT or stdin is executed once. If neither is provided, wait for input on stdin. Use --chat to start an interactive chat. Templates run once by default; use --chat to apply one interactively. Use --models to list the available model IDs without running a prompt. """ from llm.cli import ( AttachmentError, LoadTemplateError, _apply_template, _merge_template_attachments, _merge_template_options, _merge_template_tools, _run_chat, _tool_chain_kwargs, display_stream_events, load_template, logs_db_path, migrate, multi_schema, render_errors, resolve_schema_input, ) if list_models and prompt is not None: raise click.ClickException("--models cannot be used with a prompt") if list_models and template: raise click.ClickException("--models cannot be used with --template") if list_models and (tools or python_tools): raise click.ClickException("--models cannot be used with tools") if list_models and (schema_input or schema_multi): raise click.ClickException("--models cannot be used with schemas") if force_chat and prompt is not None: raise click.ClickException("--chat cannot be used with a prompt") if schema_multi: schema_input = schema_multi schema = None if schema_input: # Never create logs.db for this unlogged command. An existing # database can resolve stored schema IDs; all other schema input # is resolved using a temporary in-memory database. log_path = logs_db_path() if log_path.exists(): schema_db = sqlite_utils.Database(log_path) else: schema_db = sqlite_utils.Database(memory=True) migrate(schema_db) schema = resolve_schema_input(schema_db, schema_input, load_template) if schema_multi: schema = multi_schema(schema) template_obj = None params = dict(param) if template: try: template_obj = load_template(template) attachments, attachment_types = _merge_template_attachments( template_obj, attachments, attachment_types ) except (AttachmentError, LoadTemplateError) as ex: raise click.ClickException(str(ex)) if not model_id and template_obj.model: model_id = template_obj.model if template_obj.schema_object and not schema: schema = template_obj.schema_object if template_obj.options: options = _merge_template_options(template_obj, options) tools, python_tools = _merge_template_tools( template_obj, tools, python_tools ) if not list_models and not model_id: raise click.ClickException( "--model is required unless --models or a template model is used" ) model_class = Responses if use_responses else Chat model_kwargs = { "model_id": model_id or "", "model_name": model_id or "", "api_base": url, "headers": dict(headers), "vision": True, "audio": not use_responses, # Optimistically expose capabilities that have no effect until # the user explicitly exercises them. "reasoning": True, "verbosity": True, "image_detail_original": True, "supports_schema": True, "supports_tools": True, } if use_responses: model_kwargs["reasoning_summary"] = False model = model_class(**model_kwargs) # A configured api_base never receives the user's default OpenAI key. # Match that safety property here: only send credentials when --key # was explicitly provided for this invocation. if not key: model.needs_key = None try: validated_options = { option_name: option_value for option_name, option_value in model.Options(**dict(options)) if option_value is not None } except ValidationError as ex: raise click.ClickException(render_errors(ex.errors())) prompt_kwargs = { "options": validated_options, "schema": schema, "stream": not no_stream, "hide_reasoning": hide_reasoning, } if key: prompt_kwargs["key"] = key tool_kwargs = _tool_chain_kwargs( tools, python_tools, tools_debug, tools_approve, chain_limit, model=model, ) resolved_attachments = [*attachments, *attachment_types] try: if list_models: available_models = model.get_client(key).models.list() error = getattr(available_models, "error", None) if error: if isinstance(error, dict): error = error.get("message") or json.dumps(error) raise click.ClickException(str(error)) for available_model in available_models: click.echo(available_model.id) return if force_chat: conversation = model.conversation() def transform_chat_prompt(chat_prompt): nonlocal system if template_obj: chat_prompt, system = _apply_template( template_obj, chat_prompt, params, system ) return chat_prompt def execute_chat_prompt(chat_prompt, _fragments, turn_attachments): nonlocal system prompt_method = ( conversation.chain if tool_kwargs else conversation.prompt ) response = prompt_method( chat_prompt, system=system, attachments=turn_attachments, **prompt_kwargs, **tool_kwargs, ) system = None return response _run_chat( f"{model_id} at {url}", execute_chat_prompt, initial_attachments=resolved_attachments, transform_prompt=transform_chat_prompt, show_reasoning=not hide_reasoning, ) return if not sys.stdin.isatty(): stdin_prompt = sys.stdin.read() if stdin_prompt: prompt = " ".join( part for part in (stdin_prompt, prompt) if part is not None ) elif ( prompt is None and not resolved_attachments and not schema and (template_obj is None or "input" in template_obj.vars()) ): # Match `llm prompt`: wait for stdin until EOF instead of # implicitly starting an interactive chat. prompt = sys.stdin.read() if template_obj: prompt, system = _apply_template(template_obj, prompt, params, system) if prompt is None and not (resolved_attachments or schema): raise click.ClickException( "A prompt is required when stdin is not interactive" ) if tool_kwargs: response = model.conversation().chain( prompt, system=system, attachments=resolved_attachments, **prompt_kwargs, **tool_kwargs, ) else: response = model.prompt( prompt, system=system, attachments=resolved_attachments, **prompt_kwargs, ) display_stream_events( response.stream_events(), show_reasoning=not hide_reasoning, ) click.echo() except (click.Abort, click.ClickException): raise except (ValueError, NotImplementedError) as ex: raise click.ClickException(str(ex)) except Exception as ex: if getattr(sys, "_called_from_test", False) or os.environ.get( "LLM_RAISE_ERRORS" ): raise raise click.ClickException(str(ex)) @openai_.command() @click.option("json_", "--json", is_flag=True, help="Output as JSON") @click.option("--key", help="OpenAI API key") def models(json_, key): "List models available to you from the OpenAI API" from llm import get_key api_key = get_key(key, "openai", "OPENAI_API_KEY") response = httpx.get( "https://api.openai.com/v1/models", headers={"Authorization": f"Bearer {api_key}"}, ) if response.status_code != 200: raise click.ClickException( f"Error {response.status_code} from OpenAI API: {response.text}" ) models = response.json()["data"] if json_: click.echo(json.dumps(models, indent=4)) else: to_print = [] for model in models: # Print id, owned_by, root, created as ISO 8601 created_str = datetime.datetime.fromtimestamp( model["created"], datetime.timezone.utc ).isoformat() to_print.append( { "id": model["id"], "owned_by": model["owned_by"], "created": created_str, } ) done = dicts_to_table_string(["id", "owned_by", "created"], to_print) print("\n".join(done)) class SharedOptions(llm.Options): temperature: float | None = Field( description=( "What sampling temperature to use, between 0 and 2. Higher values like " "0.8 will make the output more random, while lower values like 0.2 will " "make it more focused and deterministic." ), ge=0, le=2, default=None, ) max_tokens: int | None = Field( description="Maximum number of tokens to generate.", default=None ) top_p: float | None = Field( description=( "An alternative to sampling with temperature, called nucleus sampling, " "where the model considers the results of the tokens with top_p " "probability mass. So 0.1 means only the tokens comprising the top " "10% probability mass are considered. Recommended to use top_p or " "temperature but not both." ), ge=0, le=1, default=None, ) frequency_penalty: float | None = Field( description=( "Number between -2.0 and 2.0. Positive values penalize new tokens based " "on their existing frequency in the text so far, decreasing the model's " "likelihood to repeat the same line verbatim." ), ge=-2, le=2, default=None, ) presence_penalty: float | None = Field( description=( "Number between -2.0 and 2.0. Positive values penalize new tokens based " "on whether they appear in the text so far, increasing the model's " "likelihood to talk about new topics." ), ge=-2, le=2, default=None, ) stop: str | None = Field( description=("A string where the API will stop generating further tokens."), default=None, ) logit_bias: dict | str | None = Field( description=( "Modify the likelihood of specified tokens appearing in the completion. " 'Pass a JSON string like \'{"1712":-100, "892":-100, "1489":-100}\'' ), default=None, ) seed: int | None = Field( description="Integer seed to attempt to sample deterministically", default=None, ) @field_validator("logit_bias") def validate_logit_bias(cls, logit_bias): if logit_bias is None: return None if isinstance(logit_bias, str): try: logit_bias = json.loads(logit_bias) except json.JSONDecodeError: raise ValueError("Invalid JSON in logit_bias string") validated_logit_bias = {} for key, value in logit_bias.items(): try: int_key = int(key) int_value = int(value) if -100 <= int_value <= 100: validated_logit_bias[int_key] = int_value else: raise ValueError("Value must be between -100 and 100") except ValueError: raise ValueError("Invalid key-value pair in logit_bias dictionary") return validated_logit_bias class ReasoningEffortEnum(str, Enum): none = "none" minimal = "minimal" low = "low" medium = "medium" high = "high" xhigh = "xhigh" max = "max" class VerbosityEnum(str, Enum): low = "low" medium = "medium" high = "high" class ImageDetailEnum(str, Enum): low = "low" high = "high" auto = "auto" class ImageDetailWithOriginalEnum(str, Enum): low = "low" high = "high" original = "original" auto = "auto" def enum_values_sentence(enum_class): values = [item.value for item in enum_class] if len(values) == 1: return values[0] return "{}, and {}".format(", ".join(values[:-1]), values[-1]) def build_options_class( *, reasoning=False, verbosity=False, image_detail_original=False, chat_completions=False, service_tier=False, ): fields = { "json_object": ( bool | None, Field( description="Output a valid JSON object {...}. Prompt must mention JSON.", default=None, ), ) } if chat_completions: fields["chat_completions"] = ( bool | None, Field( description=( "Force the use of the older /v1/chat/completions endpoint " "instead of /v1/responses. Most callers should leave this " "off; set to true to fall back to the Chat Completions code " "path for compatibility." ), default=None, ), ) image_detail_enum = ( ImageDetailWithOriginalEnum if image_detail_original else ImageDetailEnum ) image_detail_values = enum_values_sentence(image_detail_enum) fields["image_detail"] = ( image_detail_enum | None, Field( description=( "Controls the detail level for image attachments. Supported values are " f"{image_detail_values}." ), default=None, ), ) if reasoning: fields["reasoning_effort"] = ( ReasoningEffortEnum | None, Field( description=( "Constraints effort on reasoning for reasoning models. Currently " "supported values are low, medium, and high. Reducing reasoning " "effort can result in faster responses and fewer tokens used on " "reasoning in a response." ), default=None, ), ) if verbosity: fields["verbosity"] = ( VerbosityEnum | None, Field( description=( "Controls how verbose the model's response should be. Supported " "values are low, medium, and high." ), default=None, ), ) if service_tier: fields["service_tier"] = ( str | None, Field( description=( "The processing tier to use for this request - for example " "'fast' for Fast mode (faster responses at a higher price) " "or 'flex' for slower, cheaper processing on models that " "support those tiers." ), default=None, ), ) return create_model("Options", __base__=SharedOptions, **fields) def _attachment(attachment, image_detail=None): url = attachment.url base64_content = "" if not url or attachment.resolve_type().startswith("audio/"): base64_content = attachment.base64_content() url = f"data:{attachment.resolve_type()};base64,{base64_content}" if attachment.resolve_type() == "application/pdf": if not base64_content: base64_content = attachment.base64_content() return { "type": "file", "file": { "filename": f"{attachment.id()}.pdf", "file_data": f"data:application/pdf;base64,{base64_content}", }, } if attachment.resolve_type().startswith("image/"): image_url = {"url": url} if image_detail: image_url["detail"] = image_detail return {"type": "image_url", "image_url": image_url} else: format_ = "wav" if attachment.resolve_type() == "audio/wav" else "mp3" return { "type": "input_audio", "input_audio": { "data": base64_content, "format": format_, }, } class _Shared: # NEVER remove or change an existing entry - only ever append new # ones. json_replacements: ClassVar[dict] = { "completion_tokens_details_0": { "accepted_prediction_tokens": 0, "audio_tokens": 0, "reasoning_tokens": 0, "rejected_prediction_tokens": 0, }, "prompt_tokens_details_0": { "audio_tokens": 0, "cached_tokens": 0, }, } def __init__( self, model_id, key=None, model_name=None, api_base=None, api_type=None, api_version=None, api_engine=None, headers=None, can_stream=True, vision=False, audio=False, reasoning=False, verbosity=False, image_detail_original=False, service_tier=False, supports_schema=False, supports_tools=False, allows_system_prompt=True, ): self.model_id = model_id self.key = key self.supports_schema = supports_schema self.supports_tools = supports_tools self.model_name = model_name self.api_base = api_base self.api_type = api_type self.api_version = api_version self.api_engine = api_engine self.headers = headers self.can_stream = can_stream self.vision = vision self.allows_system_prompt = allows_system_prompt self.attachment_types = set() if reasoning or verbosity or image_detail_original or service_tier: self.Options = build_options_class( reasoning=reasoning, verbosity=verbosity, image_detail_original=image_detail_original, service_tier=service_tier, ) if vision: self.attachment_types.update( { "image/png", "image/jpeg", "image/webp", "image/gif", "application/pdf", } ) if audio: self.attachment_types.update( { "audio/wav", "audio/mpeg", } ) def __str__(self) -> str: return f"OpenAI Chat: {self.model_id}" def _append_llm_message(self, out, message, current_system, image_detail=None): """Translate one llm.Message into one (or more) OpenAI message dicts and append them to ``out``. Returns the (possibly updated) current_system value so the caller can avoid re-emitting an unchanged system prompt. """ from llm.parts import ( AttachmentPart, TextPart, ToolCallPart, ToolResultPart, ) text_bits = [] attachment_items = [] tool_calls = [] tool_results = [] for part in message.parts: if isinstance(part, TextPart): text_bits.append(part.text) elif isinstance(part, AttachmentPart) and part.attachment: attachment_items.append( _attachment(part.attachment, image_detail=image_detail) ) elif isinstance(part, ToolCallPart): tool_calls.append( { "type": "function", "id": part.tool_call_id, "function": { "name": part.name, "arguments": json.dumps(part.arguments), }, } ) elif isinstance(part, ToolResultPart): tool_results.append( { "role": "tool", "tool_call_id": part.tool_call_id, "content": part.output, } ) # Role "tool" emits one OpenAI "tool" message per ToolResultPart. if message.role == "tool": out.extend(tool_results) return current_system # System dedup: skip if this text is already the active system prompt. if message.role == "system": text = "".join(text_bits) if text == current_system: return current_system current_system = text if attachment_items: content = [] if text_bits: content.append({"type": "text", "text": "".join(text_bits)}) content.extend(attachment_items) entry = {"role": message.role, "content": content} else: entry = { "role": message.role, "content": "".join(text_bits) if text_bits else None, } if tool_calls: entry["tool_calls"] = tool_calls # OpenAI expects content=null when only tool_calls are present. if not text_bits: entry["content"] = None elif entry["content"] is None and message.role != "assistant": # For user/system, an empty message is pointless — drop it. return current_system out.append(entry) return current_system def build_messages(self, prompt, conversation, image_detail=None): """Translate prompt.messages into OpenAI's wire format.""" messages: list[dict[str, Any]] = [] if image_detail is not None: image_detail = image_detail.value current_system: str | None = None for msg in prompt.messages: current_system = self._append_llm_message( messages, msg, current_system, image_detail=image_detail ) return messages def set_usage(self, response, usage): if not usage: return input_tokens = usage.pop("prompt_tokens") output_tokens = usage.pop("completion_tokens") usage.pop("total_tokens") response.set_usage( input=input_tokens, output=output_tokens, details=simplify_usage_dict(usage) ) def get_client(self, key, *, async_=False): kwargs = {} if self.api_base: kwargs["base_url"] = self.api_base if self.api_type: kwargs["api_type"] = self.api_type if self.api_version: kwargs["api_version"] = self.api_version if self.api_engine: kwargs["engine"] = self.api_engine if self.needs_key: kwargs["api_key"] = self.get_key(key) else: # OpenAI-compatible models don't need a key, but the # openai client library requires one kwargs["api_key"] = "DUMMY_KEY" if self.headers: kwargs["default_headers"] = self.headers if os.environ.get("LLM_OPENAI_SHOW_RESPONSES"): kwargs["http_client"] = logging_client() if async_: return openai.AsyncOpenAI(**kwargs) else: return openai.OpenAI(**kwargs) def build_kwargs(self, prompt, stream): kwargs = dict(not_nulls(prompt.options)) json_object = kwargs.pop("json_object", None) kwargs.pop("image_detail", None) kwargs.pop("chat_completions", None) if "max_tokens" not in kwargs and self.default_max_tokens is not None: kwargs["max_tokens"] = self.default_max_tokens if json_object: kwargs["response_format"] = {"type": "json_object"} if prompt.schema: kwargs["response_format"] = { "type": "json_schema", "json_schema": {"name": "output", "schema": prompt.schema}, } if prompt.tools: kwargs["tools"] = [ { "type": "function", "function": { "name": tool.name, "description": tool.description or None, "parameters": tool.input_schema, }, } for tool in prompt.tools ] if stream: kwargs["stream_options"] = {"include_usage": True} return kwargs class Chat(_Shared, KeyModel): needs_key = "openai" key_env_var = "OPENAI_API_KEY" default_max_tokens = None Options = build_options_class() def execute( self, prompt: Prompt, stream: bool, response: Response, conversation: Conversation | None = None, key: str | None = None, ) -> Iterator[str | StreamEvent]: if prompt.system and not self.allows_system_prompt: raise NotImplementedError("Model does not support system prompts") messages = self.build_messages( prompt, conversation, image_detail=getattr(prompt.options, "image_detail", None), ) kwargs = self.build_kwargs(prompt, stream) client = self.get_client(key) usage = None if stream: completion = client.chat.completions.create( model=self.model_name or self.model_id, messages=messages, stream=True, **kwargs, ) chunks = [] tool_calls = {} for chunk in completion: chunks.append(chunk) if chunk.usage: usage = chunk.usage.model_dump() if chunk.choices and chunk.choices[0].delta: for tool_call in chunk.choices[0].delta.tool_calls or []: if tool_call.function.arguments is None: tool_call.function.arguments = "" idx = tool_call.index if idx not in tool_calls: tool_calls[idx] = tool_call yield StreamEvent( type="tool_call_name", chunk=tool_call.function.name or "", tool_call_id=tool_call.id, ) else: tool_calls[ idx ].function.arguments += tool_call.function.arguments if tool_call.function.arguments: yield StreamEvent( type="tool_call_args", chunk=tool_call.function.arguments, tool_call_id=tool_calls[idx].id, ) try: content = chunk.choices[0].delta.content except IndexError: content = None if content: # Empty strings are noise (OpenAI's first chunk # with role=assistant has content=""). yield StreamEvent(type="text", chunk=content) response.response_json = remove_dict_none_values(combine_chunks(chunks)) if tool_calls: for value in tool_calls.values(): response.add_tool_call( llm.ToolCall( tool_call_id=value.id, name=value.function.name, arguments=json.loads(value.function.arguments or "{}"), ) ) else: completion = client.chat.completions.create( model=self.model_name or self.model_id, messages=messages, stream=False, **kwargs, ) usage = completion.usage.model_dump() response.response_json = remove_dict_none_values(completion.model_dump()) for tool_call in completion.choices[0].message.tool_calls or []: response.add_tool_call( llm.ToolCall( tool_call_id=tool_call.id, name=tool_call.function.name, arguments=json.loads(tool_call.function.arguments or "{}"), ) ) yield StreamEvent( type="tool_call_name", chunk=tool_call.function.name or "", tool_call_id=tool_call.id, ) yield StreamEvent( type="tool_call_args", chunk=tool_call.function.arguments or "", tool_call_id=tool_call.id, ) if completion.choices[0].message.content is not None: yield StreamEvent( type="text", chunk=completion.choices[0].message.content, ) self.set_usage(response, usage) if usage and (usage.get("completion_tokens_details") or {}).get( "reasoning_tokens" ): yield StreamEvent(type="reasoning", chunk="", redacted=True) response._prompt_json = redact_data({"messages": messages}) class AsyncChat(_Shared, AsyncKeyModel): needs_key = "openai" key_env_var = "OPENAI_API_KEY" default_max_tokens = None Options = build_options_class() async def execute( self, prompt: Prompt, stream: bool, response: AsyncResponse, conversation: AsyncConversation | None = None, key: str | None = None, ) -> AsyncGenerator[str | StreamEvent, None]: if prompt.system and not self.allows_system_prompt: raise NotImplementedError("Model does not support system prompts") messages = self.build_messages( prompt, conversation, image_detail=getattr(prompt.options, "image_detail", None), ) kwargs = self.build_kwargs(prompt, stream) client = self.get_client(key, async_=True) usage = None if stream: completion = await client.chat.completions.create( model=self.model_name or self.model_id, messages=messages, stream=True, **kwargs, ) chunks = [] tool_calls = {} async for chunk in completion: if chunk.usage: usage = chunk.usage.model_dump() chunks.append(chunk) if chunk.choices and chunk.choices[0].delta: for tool_call in chunk.choices[0].delta.tool_calls or []: if tool_call.function.arguments is None: tool_call.function.arguments = "" idx = tool_call.index if idx not in tool_calls: tool_calls[idx] = tool_call yield StreamEvent( type="tool_call_name", chunk=tool_call.function.name or "", tool_call_id=tool_call.id, ) else: tool_calls[ idx ].function.arguments += tool_call.function.arguments if tool_call.function.arguments: yield StreamEvent( type="tool_call_args", chunk=tool_call.function.arguments, tool_call_id=tool_calls[idx].id, ) try: content = chunk.choices[0].delta.content except IndexError: content = None if content: yield StreamEvent(type="text", chunk=content) if tool_calls: for value in tool_calls.values(): response.add_tool_call( llm.ToolCall( tool_call_id=value.id, name=value.function.name, arguments=json.loads(value.function.arguments or "{}"), ) ) response.response_json = remove_dict_none_values(combine_chunks(chunks)) else: completion = await client.chat.completions.create( model=self.model_name or self.model_id, messages=messages, stream=False, **kwargs, ) response.response_json = remove_dict_none_values(completion.model_dump()) usage = completion.usage.model_dump() for tool_call in completion.choices[0].message.tool_calls or []: response.add_tool_call( llm.ToolCall( tool_call_id=tool_call.id, name=tool_call.function.name, arguments=json.loads(tool_call.function.arguments or "{}"), ) ) yield StreamEvent( type="tool_call_name", chunk=tool_call.function.name or "", tool_call_id=tool_call.id, ) yield StreamEvent( type="tool_call_args", chunk=tool_call.function.arguments or "", tool_call_id=tool_call.id, ) if completion.choices[0].message.content is not None: yield StreamEvent( type="text", chunk=completion.choices[0].message.content, ) self.set_usage(response, usage) if usage and (usage.get("completion_tokens_details") or {}).get( "reasoning_tokens" ): yield StreamEvent(type="reasoning", chunk="", redacted=True) response._prompt_json = redact_data({"messages": messages}) def _responses_attachment(attachment, image_detail=None): """Translate an llm Attachment into a Responses-API content part.""" url = attachment.url base64_content = "" if not url or attachment.resolve_type().startswith("audio/"): base64_content = attachment.base64_content() url = f"data:{attachment.resolve_type()};base64,{base64_content}" if attachment.resolve_type() == "application/pdf": if not base64_content: base64_content = attachment.base64_content() return { "type": "input_file", "filename": f"{attachment.id()}.pdf", "file_data": f"data:application/pdf;base64,{base64_content}", } if attachment.resolve_type().startswith("image/"): item = {"type": "input_image", "image_url": url} if image_detail: item["detail"] = image_detail return item # Audio is not yet supported on the Responses input shape we use; fall # back to image_url for unknown types so we don't silently drop content. return {"type": "input_image", "image_url": url} class WebSearch(llm.ServerSideTool): """Search the web using OpenAI's hosted search tool. Configure domain filters, approximate location, result context, live web access and image search through constructor arguments. Set ``include_sources`` to retain every consulted URL or ``include_results`` to retain raw results such as image search results. """ name = "web_search" _search_context_sizes = frozenset({"low", "medium", "high"}) _return_token_budgets = frozenset({"default", "unlimited"}) _search_content_types = frozenset({"text", "image"}) def __init__( self, allowed_domains: list[str] | None = None, blocked_domains: list[str] | None = None, user_location: dict | None = None, search_context_size: Literal["low", "medium", "high"] | None = None, external_web_access: bool | None = None, return_token_budget: Literal["default", "unlimited"] | None = None, search_content_types: list[Literal["text", "image"]] | None = None, image_settings: dict | None = None, include_sources: bool = False, include_results: bool = False, ): super().__init__() self.allowed_domains = self._validate_domains( "allowed_domains", allowed_domains ) self.blocked_domains = self._validate_domains( "blocked_domains", blocked_domains ) if ( search_context_size is not None and search_context_size not in self._search_context_sizes ): raise ValueError("search_context_size must be one of: low, medium or high") if external_web_access is not None and not isinstance( external_web_access, bool ): raise TypeError("external_web_access must be a boolean") if ( return_token_budget is not None and return_token_budget not in self._return_token_budgets ): raise ValueError("return_token_budget must be default or unlimited") if search_content_types is not None: if not isinstance(search_content_types, list): raise TypeError("search_content_types must be a list") invalid_content_types = set(search_content_types).difference( self._search_content_types ) if invalid_content_types: raise ValueError("search_content_types must contain text and/or image") if user_location is not None: if not isinstance(user_location, dict): raise TypeError("user_location must be a dictionary") user_location = dict(user_location) user_location.setdefault("type", "approximate") if user_location["type"] != "approximate": raise ValueError("user_location type must be approximate") if image_settings is not None: if not isinstance(image_settings, dict): raise TypeError("image_settings must be a dictionary") image_settings = dict(image_settings) max_results = image_settings.get("max_results") if max_results is not None and ( isinstance(max_results, bool) or not isinstance(max_results, int) or max_results < 1 ): raise ValueError( "image_settings max_results must be a positive integer" ) caption = image_settings.get("caption") if caption is not None and not isinstance(caption, bool): raise TypeError("image_settings caption must be a boolean") if not isinstance(include_sources, bool): raise TypeError("include_sources must be a boolean") if not isinstance(include_results, bool): raise TypeError("include_results must be a boolean") self.user_location = user_location self.search_context_size = search_context_size self.external_web_access = external_web_access self.return_token_budget = return_token_budget self.search_content_types = ( list(search_content_types) if search_content_types is not None else None ) self.image_settings = image_settings self.include_sources = include_sources self.include_results = include_results @staticmethod def _validate_domains(name, domains): if domains is None: return None if not isinstance(domains, list): raise TypeError(f"{name} must be a list") if len(domains) > 100: raise ValueError(f"{name} cannot contain more than 100 domains") for domain in domains: if not isinstance(domain, str) or not domain: raise TypeError(f"{name} entries must be non-empty strings") if domain.lower().startswith(("http://", "https://")): raise ValueError(f"{name} entries must omit the URL scheme") return list(domains) def tool_spec(self, model): spec = {"type": "web_search"} if self.allowed_domains is not None or self.blocked_domains is not None: filters = {} if self.allowed_domains is not None: filters["allowed_domains"] = list(self.allowed_domains) if self.blocked_domains is not None: filters["blocked_domains"] = list(self.blocked_domains) spec["filters"] = filters for key in ( "user_location", "search_context_size", "external_web_access", "return_token_budget", "search_content_types", "image_settings", ): value = getattr(self, key) if value is not None: if isinstance(value, dict): value = dict(value) elif isinstance(value, list): value = list(value) spec[key] = value return spec def prepare_request(self, model, kwargs): if not self.include_sources and not self.include_results: return include = kwargs.setdefault("include", []) if self.include_sources and "web_search_call.action.sources" not in include: include.append("web_search_call.action.sources") if self.include_results and "web_search_call.results" not in include: include.append("web_search_call.results") class CodeInterpreter(llm.ServerSideTool): """Run Python in an OpenAI-managed container. With no ``container`` argument OpenAI creates or reuses an automatic container. ``memory_limit`` and ``file_ids`` configure that automatic container. Pass an existing ``cntr_`` ID as ``container`` to use it explicitly instead. """ name = "code_interpreter" _memory_limits = frozenset({"1g", "4g", "16g", "64g"}) def __init__( self, container: str | None = None, memory_limit: Literal["1g", "4g", "16g", "64g"] | None = None, file_ids: list[str] | None = None, ): super().__init__() if container is not None and not isinstance(container, str): raise TypeError("container must be a string container ID") if memory_limit is not None and memory_limit not in self._memory_limits: raise ValueError("memory_limit must be one of: 1g, 4g, 16g or 64g") if container is not None and (memory_limit is not None or file_ids is not None): raise ValueError( "container cannot be combined with memory_limit or file_ids" ) self.container = container self.memory_limit = memory_limit self.file_ids = list(file_ids) if file_ids is not None else None def tool_spec(self, model): if self.container is not None: return {"type": "code_interpreter", "container": self.container} container = {"type": "auto"} if self.memory_limit is not None: container["memory_limit"] = self.memory_limit if self.file_ids is not None: container["file_ids"] = list(self.file_ids) return {"type": "code_interpreter", "container": container} def prepare_request(self, model, kwargs): include = kwargs.setdefault("include", []) if "code_interpreter_call.outputs" not in include: include.append("code_interpreter_call.outputs") class _SharedResponses(_Shared): """Mixin that translates llm.Prompt into Responses API parameters.""" @property def supported_server_side_tools(self): return (WebSearch, CodeInterpreter, llm.ServerSideTool) # Recurring boilerplate in Responses API payloads. Same contract as # _Shared.json_replacements, which this replaces for Responses # models: NEVER remove or change an existing entry - only ever # append new ones. json_replacements: ClassVar[dict] = { "tool_usage_0": { "image_gen": { "input_tokens": 0, "input_tokens_details": { "image_tokens": 0, "text_tokens": 0, }, "output_tokens": 0, "output_tokens_details": { "image_tokens": 0, "text_tokens": 0, }, "total_tokens": 0, }, "web_search": {"num_requests": 0}, }, "input_tokens_details_0": { "cached_tokens": 0, "cache_write_tokens": 0, }, "reasoning_settings_0": { "effort": "medium", "summary": "detailed", "context": "all_turns", "mode": "standard", }, "reasoning_settings_1": { "effort": "medium", "summary": "detailed", "context": "current_turn", "mode": "standard", }, # The default text block on non-schema replies "text_format_0": {"format": {"type": "text"}, "verbosity": "medium"}, # The static envelope of a Responses payload "response_env_0": { "object": "response", "parallel_tool_calls": True, "temperature": 1.0, "tool_choice": "auto", "top_p": 1.0, "background": False, "service_tier": "default", "status": "completed", "top_logprobs": 0, "truncation": "disabled", "frequency_penalty": 0.0, "presence_penalty": 0.0, "store": False, "tools": [], }, "message_completed": { "role": "assistant", "status": "completed", "type": "message", "phase": "final_answer", }, } def __str__(self) -> str: return f"OpenAI Responses: {self.model_id}" def _delegate_chat_kwargs(self): """Return constructor kwargs that mirror this Responses model so we can build a sibling Chat / AsyncChat instance for the ``-o chat_completions 1`` opt-out path.""" return { "model_id": self.model_id, "key": self.key, "model_name": self.model_name, "api_base": self.api_base, "api_type": self.api_type, "api_version": self.api_version, "api_engine": self.api_engine, "headers": self.headers, "can_stream": self.can_stream, "vision": self.vision, "reasoning": self._reasoning, "verbosity": self._verbosity, "image_detail_original": self._image_detail_original, "service_tier": self._service_tier, "supports_schema": self.supports_schema, "supports_tools": self.supports_tools, "allows_system_prompt": self.allows_system_prompt, } def _build_responses_input(self, prompt, image_detail=None): """Translate prompt.messages into a (input_items, instructions) tuple for the Responses API. The most recent system Message is hoisted into ``instructions``; earlier system messages are dropped (mirroring the way the Chat path collapses repeated identical system prompts). """ from llm.parts import ( AttachmentPart, ReasoningPart, TextPart, ToolCallPart, ToolResultPart, ) items: list[dict[str, Any]] = [] instructions: str | None = None for msg in prompt.messages: if msg.role == "system": text = "".join(p.text for p in msg.parts if isinstance(p, TextPart)) if text: instructions = text continue text_bits: list[str] = [] attachment_items: list[dict[str, Any]] = [] tool_call_items: list[dict[str, Any]] = [] tool_result_items: list[dict[str, Any]] = [] reasoning_items: list[dict[str, Any]] = [] for part in msg.parts: if isinstance(part, TextPart): text_bits.append(part.text) elif isinstance(part, AttachmentPart) and part.attachment: attachment_items.append( _responses_attachment( part.attachment, image_detail=image_detail ) ) elif isinstance(part, ToolCallPart): if part.server_executed: # Server-side tool calls (web_search, # code_interpreter) ran inside OpenAI's # infrastructure - they must not be replayed as # client function_call items. continue tool_call_items.append( { "type": "function_call", "call_id": part.tool_call_id, "name": part.name, "arguments": json.dumps(part.arguments), } ) elif isinstance(part, ToolResultPart): if part.server_executed: continue tool_result_items.append( { "type": "function_call_output", "call_id": part.tool_call_id, "output": part.output, } ) elif isinstance(part, ReasoningPart): pm = (part.provider_metadata or {}).get("openai") or {} enc = pm.get("encrypted_content") rid = pm.get("id") if enc or rid: # Round-trip a previous reasoning item so the model # can pick up where it left off mid-tool-call. item: dict[str, Any] = {"type": "reasoning"} if rid: item["id"] = rid if enc: item["encrypted_content"] = enc if pm.get("summary"): item["summary"] = pm["summary"] else: item["summary"] = [] reasoning_items.append(item) # Reasoning items must precede the assistant message / function # call they belonged to. items.extend(reasoning_items) if msg.role == "tool": items.extend(tool_result_items) continue if msg.role == "user": if attachment_items: content: list[dict[str, Any]] = [] if text_bits: content.append( {"type": "input_text", "text": "".join(text_bits)} ) content.extend(attachment_items) items.append({"role": "user", "content": content}) elif text_bits: items.append({"role": "user", "content": "".join(text_bits)}) elif msg.role == "assistant": if text_bits: items.append({"role": "assistant", "content": "".join(text_bits)}) items.extend(tool_call_items) return items, instructions def _build_responses_kwargs(self, prompt, stream): """Build the keyword arguments for client.responses.create().""" opts = dict(not_nulls(prompt.options)) # Strip options that are either internal to llm or not accepted by # the Responses API. opts.pop("json_object", None) opts.pop("chat_completions", None) opts.pop("image_detail", None) max_tokens = opts.pop("max_tokens", None) reasoning_effort = opts.pop("reasoning_effort", None) verbosity = opts.pop("verbosity", None) temperature = opts.pop("temperature", None) top_p = opts.pop("top_p", None) seed = opts.pop("seed", None) kwargs: dict[str, Any] = {} if max_tokens is None and self.default_max_tokens is not None: max_tokens = self.default_max_tokens if max_tokens is not None: kwargs["max_output_tokens"] = max_tokens if temperature is not None: kwargs["temperature"] = temperature if top_p is not None: kwargs["top_p"] = top_p if seed is not None: kwargs["seed"] = seed if self._reasoning: reasoning = {} if self._reasoning_summary and not getattr(prompt, "hide_reasoning", False): reasoning["summary"] = "auto" if reasoning_effort: reasoning["effort"] = reasoning_effort if reasoning: kwargs["reasoning"] = reasoning text: dict[str, Any] = {} if verbosity: text["verbosity"] = verbosity if prompt.options.json_object: text["format"] = {"type": "json_object"} if prompt.schema: # ``strict: False`` mirrors the looser behaviour of the # /v1/chat/completions json_schema response_format - required # because the Responses API otherwise insists on # ``additionalProperties: false`` everywhere. text["format"] = { "type": "json_schema", "name": "output", "schema": prompt.schema, "strict": False, } if text: kwargs["text"] = text if prompt.tools: _partition_tools(self, prompt.tools) kwargs["tools"] = [ ( { "type": "function", "name": tool.name, "description": tool.description or None, "parameters": tool.input_schema, } if isinstance(tool, llm.Tool) else tool.tool_spec(self) ) for tool in prompt.tools ] # Pass anything we did not consume through verbatim - this lets # extras like ``parallel_tool_calls`` flow into the API. kwargs.update(opts) return kwargs def _finalize_responses_kwargs(self, prompt, stream, instructions=None): """Build complete request kwargs, then run server-tool hooks in order.""" kwargs = self._build_responses_kwargs(prompt, stream) if instructions is not None: kwargs["instructions"] = instructions kwargs["store"] = False if self._reasoning and ( self._reasoning_summary or getattr(prompt.options, "reasoning_effort", None) ): include = kwargs.setdefault("include", []) if "reasoning.encrypted_content" not in include: include.append("reasoning.encrypted_content") _, server_side_tools = _partition_tools(self, prompt.tools) for tool in server_side_tools: tool.prepare_request(self, kwargs) return kwargs def _set_usage_responses(self, response, usage): if not usage: return input_tokens = usage.get("input_tokens", 0) or 0 output_tokens = usage.get("output_tokens", 0) or 0 details = {} for key in ("input_tokens_details", "output_tokens_details"): value = usage.get(key) if value: details[key] = value response.set_usage( input=input_tokens, output=output_tokens, details=details or None ) def _reasoning_text_from_item(self, item): bits = [] for attr in ("summary", "content"): for part in getattr(item, attr, None) or []: if isinstance(part, dict): text = part.get("text") else: text = getattr(part, "text", None) if text: bits.append(text) return "".join(bits) def _reasoning_event(self, item, *, include_text=True): """Build a redacted-reasoning StreamEvent that carries the opaque ``id`` and ``encrypted_content`` from a Responses-API reasoning item. Echoing this metadata back on the next request via ``_build_responses_input`` lets the model pick up its prior chain of thought - critical for tool-using reasoning models, since without it the model loses ~3% on SWE-bench (per OpenAI).""" rid = getattr(item, "id", None) enc = getattr(item, "encrypted_content", None) summary = getattr(item, "summary", None) text = self._reasoning_text_from_item(item) if include_text else "" meta: dict[str, Any] = {} if rid: meta["id"] = rid if enc: meta["encrypted_content"] = enc if summary: # ``summary`` is a list of {type:"summary_text", text:"..."} # objects when reasoning summaries are enabled. try: meta["summary"] = [ s.model_dump() if hasattr(s, "model_dump") else dict(s) for s in summary ] except Exception: # noqa: BLE001 meta["summary"] = list(summary) return StreamEvent( type="reasoning", chunk=text, redacted=include_text and not text, provider_metadata={"openai": meta} if meta else None, ) def _reasoning_refresh_events(self, response_json, done_events): """Metadata-only reasoning events rebuilt from the final payload. While streaming, reasoning metadata is first harvested from the ``response.output_item.done`` event, but the ``response.completed`` payload carries a *different* ciphertext of the same reasoning - OpenAI encrypts per event. Re-emitting the metadata from the final payload, aimed at the already-resolved part_index, makes the stored part and ``response_json`` agree on one blob (which also lets the log store condense the payload against the part). ``done_events`` maps reasoning item id to the StreamEvent yielded at ``output_item.done``; the framework has resolved ``part_index`` on it by the time the stream ends. """ events = [] for item in response_json.get("output") or []: if not isinstance(item, dict) or item.get("type") != "reasoning": continue prior = done_events.get(item.get("id")) if prior is None or prior.part_index is None: continue meta = { key: item[key] for key in ("id", "encrypted_content", "summary") if item.get(key) } if meta: events.append( StreamEvent( type="reasoning", chunk="", part_index=prior.part_index, provider_metadata={"openai": meta}, message_index=prior.message_index, ) ) return events def _server_tool_events(self, item, message_index): """StreamEvents for a server-side tool call output item (web_search_call / code_interpreter_call), or [] for other item types. The call and its result both carry ``server_executed=True`` so they are recorded in the message parts without entering the locally-executable tool call list. """ item_type = getattr(item, "type", None) item_id = getattr(item, "id", None) events: list[StreamEvent] = [] if item_type == "web_search_call": action = getattr(item, "action", None) if hasattr(action, "model_dump"): action = action.model_dump() events.append( StreamEvent( type="tool_call_name", chunk="web_search", tool_call_id=item_id, server_executed=True, message_index=message_index, ) ) events.append( StreamEvent( type="tool_call_args", chunk=json.dumps(action or {}), tool_call_id=item_id, server_executed=True, message_index=message_index, ) ) results = getattr(item, "results", None) or [] results = [ result.model_dump() if hasattr(result, "model_dump") else result for result in results ] events.append( StreamEvent( type="tool_result", chunk=( json.dumps(results) if results else (getattr(item, "status", None) or "completed") ), tool_call_id=item_id, server_executed=True, tool_name="web_search", message_index=message_index, ) ) elif item_type == "code_interpreter_call": code = getattr(item, "code", None) or "" events.append( StreamEvent( type="tool_call_name", chunk="code_interpreter", tool_call_id=item_id, server_executed=True, message_index=message_index, ) ) events.append( StreamEvent( type="tool_call_args", chunk=json.dumps({"code": code}), tool_call_id=item_id, server_executed=True, message_index=message_index, ) ) output_bits = [] for output in getattr(item, "outputs", None) or []: if hasattr(output, "model_dump"): output = output.model_dump() if isinstance(output, dict): text = output.get("logs") or output.get("url") if text: output_bits.append(text) events.append( StreamEvent( type="tool_result", chunk="\n".join(output_bits) or (getattr(item, "status", None) or "completed"), tool_call_id=item_id, server_executed=True, tool_name="code_interpreter", message_index=message_index, ) ) return events def _refresh_server_tool_events(self, output, done_events): """Replace streamed server-tool payloads with their final values. OpenAI can return incomplete sources, results or outputs on a ``response.output_item.done`` event and then provide the complete item on ``response.completed``. The response stores yielded StreamEvent objects by reference, so updating their chunks here corrects the assembled Parts without emitting duplicate events. """ for item in output or []: item_id = getattr(item, "id", None) prior_events = done_events.get(item_id) if not prior_events: continue final_events = { event.type: event for event in self._server_tool_events( item, prior_events[0].message_index ) } for prior_event in prior_events: final_event = final_events.get(prior_event.type) if final_event is not None: prior_event.chunk = final_event.chunk def _non_streaming_output_events(self, output, response): """Translate a non-streaming Responses ``output`` item list into StreamEvents. Returns ``(events, had_reasoning)``. Each ``message`` item after the first starts a new ``message_index``, so server-side tool execution that interleaves multiple message items assembles into multiple assistant Messages. Items between two message items (tool calls, reasoning) group with the preceding message. """ events: list[StreamEvent] = [] had_reasoning = False message_index = 0 seen_message = False for item in output: if item.type == "message" and seen_message: message_index += 1 if item.type == "reasoning": had_reasoning = True event = self._reasoning_event(item) event.message_index = message_index events.append(event) elif item.type == "function_call": try: args = json.loads(item.arguments) if item.arguments else {} except json.JSONDecodeError: args = {"_raw": item.arguments} response.add_tool_call( llm.ToolCall( tool_call_id=item.call_id, name=item.name, arguments=args, ) ) events.append( StreamEvent( type="tool_call_name", chunk=item.name or "", tool_call_id=item.call_id, message_index=message_index, ) ) events.append( StreamEvent( type="tool_call_args", chunk=item.arguments or "", tool_call_id=item.call_id, message_index=message_index, ) ) elif item.type == "message": seen_message = True for content in item.content or []: ctype = getattr(content, "type", None) if ctype == "output_text" and content.text: events.append( StreamEvent( type="text", chunk=content.text, message_index=message_index, ) ) else: events.extend(self._server_tool_events(item, message_index)) return events, had_reasoning class Responses(_SharedResponses, KeyModel): needs_key = "openai" key_env_var = "OPENAI_API_KEY" default_max_tokens = None def __init__( self, model_id, key=None, model_name=None, api_base=None, api_type=None, api_version=None, api_engine=None, headers=None, can_stream=True, vision=False, audio=False, reasoning=False, verbosity=False, image_detail_original=False, service_tier=False, supports_schema=False, supports_tools=False, allows_system_prompt=True, reasoning_summary=True, ): super().__init__( model_id, key=key, model_name=model_name, api_base=api_base, api_type=api_type, api_version=api_version, api_engine=api_engine, headers=headers, can_stream=can_stream, vision=vision, audio=audio, reasoning=reasoning, verbosity=verbosity, image_detail_original=image_detail_original, service_tier=service_tier, supports_schema=supports_schema, supports_tools=supports_tools, allows_system_prompt=allows_system_prompt, ) self._reasoning = reasoning self._reasoning_summary = reasoning_summary self._verbosity = verbosity self._image_detail_original = image_detail_original self._service_tier = service_tier # Override the Options class so that ``-o chat_completions 1`` is # always available on Responses-routed models. self.Options = build_options_class( reasoning=reasoning, verbosity=verbosity, image_detail_original=image_detail_original, chat_completions=True, service_tier=service_tier, ) def execute( self, prompt: Prompt, stream: bool, response: Response, conversation: Conversation | None = None, key: str | None = None, ) -> Iterator[str | StreamEvent]: if getattr(prompt.options, "chat_completions", None): chat = Chat(**self._delegate_chat_kwargs()) _partition_tools(chat, prompt.tools) yield from chat.execute(prompt, stream, response, conversation, key) return if prompt.system and not self.allows_system_prompt: raise NotImplementedError("Model does not support system prompts") image_detail = getattr(prompt.options, "image_detail", None) if image_detail is not None: image_detail = image_detail.value input_items, instructions = self._build_responses_input( prompt, image_detail=image_detail ) kwargs = self._finalize_responses_kwargs(prompt, stream, instructions) client = self.get_client(key) usage = None had_reasoning = False if stream: stream_obj = client.responses.create( model=self.model_name or self.model_id, input=input_items, stream=True, **kwargs, ) tool_call_meta: dict[str, dict[str, str]] = {} final_response_dict: dict[str, Any] | None = None reasoning_items_with_streamed_text = set() reasoning_done_events: dict[str, StreamEvent] = {} server_tool_done_events: dict[str, list[StreamEvent]] = {} message_index = 0 seen_message = False for event in stream_obj: etype = getattr(event, "type", None) if etype == "response.output_item.added": item = event.item if item.type == "message": if seen_message: message_index += 1 seen_message = True elif item.type == "function_call": tool_call_meta[item.id] = { "id": item.id, "call_id": item.call_id, "name": item.name, } yield StreamEvent( type="tool_call_name", chunk=item.name or "", tool_call_id=item.call_id, message_index=message_index, ) elif etype == "response.output_text.delta": yield StreamEvent( type="text", chunk=event.delta or "", message_index=message_index, ) elif etype == "response.function_call_arguments.delta": item_id = getattr(event, "item_id", None) meta = tool_call_meta.get(item_id) if item_id else None call_id = meta["call_id"] if meta else None yield StreamEvent( type="tool_call_args", chunk=event.delta or "", tool_call_id=call_id, message_index=message_index, ) elif etype in ( "response.reasoning_summary_text.delta", "response.reasoning_text.delta", ): item_id = getattr(event, "item_id", None) if item_id: reasoning_items_with_streamed_text.add(item_id) yield StreamEvent( type="reasoning", chunk=event.delta or "", message_index=message_index, ) elif etype in ( "response.reasoning_summary_text.done", "response.reasoning_text.done", ): item_id = getattr(event, "item_id", None) if item_id not in reasoning_items_with_streamed_text: text = getattr(event, "text", None) or "" if text: if item_id: reasoning_items_with_streamed_text.add(item_id) yield StreamEvent( type="reasoning", chunk=text, message_index=message_index, ) elif etype == "response.output_item.done": item = event.item if item.type == "reasoning": had_reasoning = True item_id = getattr(item, "id", None) reasoning_event = self._reasoning_event( item, include_text=( item_id not in reasoning_items_with_streamed_text ), ) reasoning_event.message_index = message_index if item_id: # Retained so the refresh after # response.completed can target the part # this event resolved to. reasoning_done_events[item_id] = reasoning_event yield reasoning_event elif item.type == "function_call": try: args = json.loads(item.arguments) if item.arguments else {} except json.JSONDecodeError: args = {"_raw": item.arguments} response.add_tool_call( llm.ToolCall( tool_call_id=item.call_id, name=item.name, arguments=args, ) ) else: server_events = self._server_tool_events(item, message_index) item_id = getattr(item, "id", None) if item_id and server_events: server_tool_done_events[item_id] = server_events yield from server_events elif etype == "response.completed": self._refresh_server_tool_events( event.response.output, server_tool_done_events ) final_response_dict = event.response.model_dump(warnings=False) if final_response_dict.get("usage"): usage = final_response_dict["usage"] if final_response_dict is not None: response.response_json = remove_dict_none_values(final_response_dict) yield from self._reasoning_refresh_events( response.response_json, reasoning_done_events ) else: completion = client.responses.create( model=self.model_name or self.model_id, input=input_items, stream=False, **kwargs, ) dumped = completion.model_dump(warnings=False) response.response_json = remove_dict_none_values(dumped) usage = dumped.get("usage") events, had_reasoning = self._non_streaming_output_events( completion.output, response ) yield from events self._set_usage_responses(response, usage) # Fallback: usage said reasoning happened but the API gave us no # reasoning items to harvest encrypted_content from. Emit the # opaque "reasoning happened" marker for UI / token accounting. if ( not had_reasoning and usage and ((usage.get("output_tokens_details") or {}).get("reasoning_tokens")) ): yield StreamEvent(type="reasoning", chunk="", redacted=True) response._prompt_json = redact_data( {"input": input_items, "instructions": instructions} ) class AsyncResponses(_SharedResponses, AsyncKeyModel): needs_key = "openai" key_env_var = "OPENAI_API_KEY" default_max_tokens = None def __init__( self, model_id, key=None, model_name=None, api_base=None, api_type=None, api_version=None, api_engine=None, headers=None, can_stream=True, vision=False, audio=False, reasoning=False, verbosity=False, image_detail_original=False, service_tier=False, supports_schema=False, supports_tools=False, allows_system_prompt=True, reasoning_summary=True, ): super().__init__( model_id, key=key, model_name=model_name, api_base=api_base, api_type=api_type, api_version=api_version, api_engine=api_engine, headers=headers, can_stream=can_stream, vision=vision, audio=audio, reasoning=reasoning, verbosity=verbosity, image_detail_original=image_detail_original, service_tier=service_tier, supports_schema=supports_schema, supports_tools=supports_tools, allows_system_prompt=allows_system_prompt, ) self._reasoning = reasoning self._reasoning_summary = reasoning_summary self._verbosity = verbosity self._image_detail_original = image_detail_original self._service_tier = service_tier self.Options = build_options_class( reasoning=reasoning, verbosity=verbosity, image_detail_original=image_detail_original, chat_completions=True, service_tier=service_tier, ) async def execute( self, prompt: Prompt, stream: bool, response: AsyncResponse, conversation: AsyncConversation | None = None, key: str | None = None, ) -> AsyncGenerator[str | StreamEvent, None]: if getattr(prompt.options, "chat_completions", None): chat = AsyncChat(**self._delegate_chat_kwargs()) _partition_tools(chat, prompt.tools) async for event in chat.execute( prompt, stream, response, conversation, key ): yield event return if prompt.system and not self.allows_system_prompt: raise NotImplementedError("Model does not support system prompts") image_detail = getattr(prompt.options, "image_detail", None) if image_detail is not None: image_detail = image_detail.value input_items, instructions = self._build_responses_input( prompt, image_detail=image_detail ) kwargs = self._finalize_responses_kwargs(prompt, stream, instructions) client = self.get_client(key, async_=True) usage = None had_reasoning = False if stream: stream_obj = await client.responses.create( model=self.model_name or self.model_id, input=input_items, stream=True, **kwargs, ) tool_call_meta: dict[str, dict[str, str]] = {} final_response_dict: dict[str, Any] | None = None reasoning_items_with_streamed_text = set() reasoning_done_events: dict[str, StreamEvent] = {} server_tool_done_events: dict[str, list[StreamEvent]] = {} message_index = 0 seen_message = False async for event in stream_obj: etype = getattr(event, "type", None) if etype == "response.output_item.added": item = event.item if item.type == "message": if seen_message: message_index += 1 seen_message = True elif item.type == "function_call": tool_call_meta[item.id] = { "id": item.id, "call_id": item.call_id, "name": item.name, } yield StreamEvent( type="tool_call_name", chunk=item.name or "", tool_call_id=item.call_id, message_index=message_index, ) elif etype == "response.output_text.delta": yield StreamEvent( type="text", chunk=event.delta or "", message_index=message_index, ) elif etype == "response.function_call_arguments.delta": item_id = getattr(event, "item_id", None) meta = tool_call_meta.get(item_id) if item_id else None call_id = meta["call_id"] if meta else None yield StreamEvent( type="tool_call_args", chunk=event.delta or "", tool_call_id=call_id, message_index=message_index, ) elif etype in ( "response.reasoning_summary_text.delta", "response.reasoning_text.delta", ): item_id = getattr(event, "item_id", None) if item_id: reasoning_items_with_streamed_text.add(item_id) yield StreamEvent( type="reasoning", chunk=event.delta or "", message_index=message_index, ) elif etype in ( "response.reasoning_summary_text.done", "response.reasoning_text.done", ): item_id = getattr(event, "item_id", None) if item_id not in reasoning_items_with_streamed_text: text = getattr(event, "text", None) or "" if text: if item_id: reasoning_items_with_streamed_text.add(item_id) yield StreamEvent( type="reasoning", chunk=text, message_index=message_index, ) elif etype == "response.output_item.done": item = event.item if item.type == "reasoning": had_reasoning = True item_id = getattr(item, "id", None) reasoning_event = self._reasoning_event( item, include_text=( item_id not in reasoning_items_with_streamed_text ), ) reasoning_event.message_index = message_index if item_id: # Retained so the refresh after # response.completed can target the part # this event resolved to. reasoning_done_events[item_id] = reasoning_event yield reasoning_event elif item.type == "function_call": try: args = json.loads(item.arguments) if item.arguments else {} except json.JSONDecodeError: args = {"_raw": item.arguments} response.add_tool_call( llm.ToolCall( tool_call_id=item.call_id, name=item.name, arguments=args, ) ) else: server_events = self._server_tool_events(item, message_index) item_id = getattr(item, "id", None) if item_id and server_events: server_tool_done_events[item_id] = server_events for server_event in server_events: yield server_event elif etype == "response.completed": self._refresh_server_tool_events( event.response.output, server_tool_done_events ) final_response_dict = event.response.model_dump(warnings=False) if final_response_dict.get("usage"): usage = final_response_dict["usage"] if final_response_dict is not None: response.response_json = remove_dict_none_values(final_response_dict) for refresh in self._reasoning_refresh_events( response.response_json, reasoning_done_events ): yield refresh else: completion = await client.responses.create( model=self.model_name or self.model_id, input=input_items, stream=False, **kwargs, ) dumped = completion.model_dump(warnings=False) response.response_json = remove_dict_none_values(dumped) usage = dumped.get("usage") events, had_reasoning = self._non_streaming_output_events( completion.output, response ) for event in events: yield event self._set_usage_responses(response, usage) if ( not had_reasoning and usage and ((usage.get("output_tokens_details") or {}).get("reasoning_tokens")) ): yield StreamEvent(type="reasoning", chunk="", redacted=True) response._prompt_json = redact_data( {"input": input_items, "instructions": instructions} ) class Completion(Chat): class Options(SharedOptions): logprobs: int | None = Field( description="Include the log probabilities of most likely N per token", default=None, le=5, ) def __init__(self, *args, default_max_tokens=None, **kwargs): super().__init__(*args, **kwargs) self.default_max_tokens = default_max_tokens def __str__(self) -> str: return f"OpenAI Completion: {self.model_id}" def execute( self, prompt: Prompt, stream: bool, response: Response, conversation: Conversation | None = None, key: str | None = None, ) -> Iterator[str | StreamEvent]: if prompt.system: raise NotImplementedError( "System prompts are not supported for OpenAI completion models" ) from llm.parts import TextPart # prompt.messages carries the full history - including history # reloaded from storage, which conversation.responses does not. messages = [] for message in prompt.messages: if message.role not in ("user", "assistant"): continue text = "".join( part.text for part in message.parts if isinstance(part, TextPart) and part.text ) if text: messages.append(text) kwargs = self.build_kwargs(prompt, stream) client = self.get_client(key) if stream: completion = client.completions.create( model=self.model_name or self.model_id, prompt="\n".join(messages), stream=True, **kwargs, ) chunks = [] for chunk in completion: chunks.append(chunk) try: content = chunk.choices[0].text except IndexError: content = None if content is not None: yield content combined = combine_chunks(chunks) cleaned = remove_dict_none_values(combined) response.response_json = cleaned else: completion = client.completions.create( model=self.model_name or self.model_id, prompt="\n".join(messages), stream=False, **kwargs, ) response.response_json = remove_dict_none_values(completion.model_dump()) yield completion.choices[0].text response._prompt_json = redact_data({"messages": messages}) def not_nulls(data) -> dict: return {key: value for key, value in data if value is not None} def combine_chunks(chunks: list) -> dict: content = "" role = None finish_reason = None # If any of them have log probability, we're going to persist # those later on logprobs = [] usage = {} for item in chunks: if item.usage: usage = item.usage.model_dump() for choice in item.choices: if choice.logprobs and hasattr(choice.logprobs, "top_logprobs"): logprobs.append( { "text": choice.text if hasattr(choice, "text") else None, "top_logprobs": choice.logprobs.top_logprobs, } ) if not hasattr(choice, "delta"): content += choice.text continue role = choice.delta.role if choice.delta.content is not None: content += choice.delta.content if choice.finish_reason is not None: finish_reason = choice.finish_reason # Imitations of the OpenAI API may be missing some of these fields combined = { "content": content, "role": role, "finish_reason": finish_reason, "usage": usage, } if logprobs: combined["logprobs"] = logprobs if chunks: for key in ("id", "object", "model", "created", "index"): value = getattr(chunks[0], key, None) if value is not None: combined[key] = value return combined def redact_data(input_dict): """ Recursively search through the input dictionary for any 'image_url' keys and modify the 'url' value to be just 'data:...'. Also redact input_audio.data keys """ if isinstance(input_dict, dict): for key, value in input_dict.items(): if ( key == "image_url" and isinstance(value, dict) and "url" in value and value["url"].startswith("data:") ): value["url"] = "data:..." elif key == "input_audio" and isinstance(value, dict) and "data" in value: value["data"] = "..." else: redact_data(value) elif isinstance(input_dict, list): for item in input_dict: redact_data(item) return input_dict ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/embeddings.py0000644000175100017510000003052715234420023015131 0ustar00runnerrunnerimport hashlib import json import time from collections.abc import Iterable from dataclasses import dataclass from itertools import islice from typing import Any, cast from sqlite_utils import Database from sqlite_utils.db import Table from .embeddings_migrations import embeddings_migrations from .models import EmbeddingModel @dataclass class Entry: id: str score: float | None content: str | None = None metadata: dict[str, Any] | None = None class Collection: class DoesNotExist(Exception): pass def __init__( self, name: str, db: Database | None = None, *, model: EmbeddingModel | None = None, model_id: str | None = None, create: bool = True, ) -> None: """ A collection of embeddings Returns the collection with the given name, creating it if it does not exist. If you set create=False a Collection.DoesNotExist exception will be raised if the collection does not already exist. Args: db (sqlite_utils.Database): Database to store the collection in name (str): Name of the collection model (llm.models.EmbeddingModel, optional): Embedding model to use model_id (str, optional): Alternatively, ID of the embedding model to use create (bool, optional): Whether to create the collection if it does not exist """ import llm self.db = db or Database(memory=True) self.name = name self._model = model embeddings_migrations.apply(self.db) rows = list(self.db["collections"].rows_where("name = ?", [self.name])) if rows: row = rows[0] self.id = row["id"] self.model_id = row["model"] else: if create: # Collection does not exist, so model or model_id is required if not model and not model_id: raise ValueError( "Either model= or model_id= must be provided when creating a new collection" ) # Create it if model_id: # Resolve alias model = llm.get_embedding_model(model_id) self._model = model model_id = cast(EmbeddingModel, model).model_id self.id = ( cast(Table, self.db["collections"]) .insert( { "name": self.name, "model": model_id, } ) .last_pk ) else: raise self.DoesNotExist(f"Collection '{name}' does not exist") def model(self) -> EmbeddingModel: "Return the embedding model used by this collection" import llm if self._model is None: self._model = llm.get_embedding_model(self.model_id) return cast(EmbeddingModel, self._model) def count(self) -> int: """ Count the number of items in the collection. Returns: int: Number of items in the collection """ return next( self.db.query( """ select count(*) as c from embeddings where collection_id = ( select id from collections where name = ? ) """, (self.name,), ) )["c"] def embed( self, id: str, value: str | bytes, metadata: dict[str, Any] | None = None, store: bool = False, ) -> None: """ Embed value and store it in the collection with a given ID. Args: id (str): ID for the value value (str or bytes): value to be embedded metadata (dict, optional): Metadata to be stored store (bool, optional): Whether to store the value in the content or content_blob column """ from llm import encode content_hash = self.content_hash(value) if self.db["embeddings"].count_where( "content_hash = ? and collection_id = ?", [content_hash, self.id] ): return embedding = self.model().embed(value) cast(Table, self.db["embeddings"]).insert( { "collection_id": self.id, "id": id, "embedding": encode(embedding), "content": value if (store and isinstance(value, str)) else None, "content_blob": value if (store and isinstance(value, bytes)) else None, "content_hash": content_hash, "metadata": json.dumps(metadata) if metadata else None, "updated": int(time.time()), }, replace=True, ) def embed_multi( self, entries: Iterable[tuple[str, str | bytes]], store: bool = False, batch_size: int = 100, ) -> None: """ Embed multiple texts and store them in the collection with given IDs. Args: entries (iterable): Iterable of (id: str, text: str) tuples store (bool, optional): Whether to store the text in the content column batch_size (int, optional): custom maximum batch size to use """ self.embed_multi_with_metadata( ((id, value, None) for id, value in entries), store=store, batch_size=batch_size, ) def embed_multi_with_metadata( self, entries: Iterable[tuple[str, str | bytes, dict[str, Any] | None]], store: bool = False, batch_size: int = 100, ) -> None: """ Embed multiple values along with metadata and store them in the collection with given IDs. Args: entries (iterable): Iterable of (id: str, value: str or bytes, metadata: None or dict) store (bool, optional): Whether to store the value in the content or content_blob column batch_size (int, optional): custom maximum batch size to use """ import llm batch_size = min(batch_size, (self.model().batch_size or batch_size)) iterator = iter(entries) collection_id = self.id while True: batch = list(islice(iterator, batch_size)) if not batch: break # Calculate hashes first items_and_hashes = [(item, self.content_hash(item[1])) for item in batch] # Any of those hashes already exist? existing_ids = [ row["id"] for row in self.db.query( """ select id from embeddings where collection_id = ? and content_hash in ({}) """.format(",".join("?" for _ in items_and_hashes)), [collection_id] + [item_and_hash[1] for item_and_hash in items_and_hashes], ) ] filtered_batch = [item for item in batch if item[0] not in existing_ids] embeddings = list( self.model().embed_multi(item[1] for item in filtered_batch) ) with self.db.atomic(): cast(Table, self.db["embeddings"]).insert_all( ( { "collection_id": collection_id, "id": id, "embedding": llm.encode(embedding), "content": ( value if (store and isinstance(value, str)) else None ), "content_blob": ( value if (store and isinstance(value, bytes)) else None ), "content_hash": self.content_hash(value), "metadata": json.dumps(metadata) if metadata else None, "updated": int(time.time()), } for (embedding, (id, value, metadata)) in zip( embeddings, filtered_batch ) ), replace=True, ) def similar_by_vector( self, vector: list[float], number: int = 10, skip_id: str | None = None, prefix: str | None = None, ) -> list[Entry]: """ Find similar items in the collection by a given vector. Args: vector (list): Vector to search by number (int, optional): Number of similar items to return skip_id (str, optional): An ID to exclude from the results prefix: (str, optional): Filter results to IDs witih this prefix Returns: list: List of Entry objects """ import llm def distance_score(other_encoded): other_vector = llm.decode(other_encoded) return llm.cosine_similarity(other_vector, vector) self.db.register_function(distance_score, replace=True) where_bits = ["collection_id = ?"] where_args = [str(self.id)] if prefix: where_bits.append("id LIKE ? || '%'") where_args.append(prefix) if skip_id: where_bits.append("id != ?") where_args.append(skip_id) return [ Entry( id=row["id"], score=row["score"], content=row["content"], metadata=json.loads(row["metadata"]) if row["metadata"] else None, ) for row in self.db.query( """ select id, content, metadata, distance_score(embedding) as score from embeddings where {where} order by score desc limit {number} """.format( where=" and ".join(where_bits), number=number, ), where_args, ) ] def similar_by_id( self, id: str, number: int = 10, prefix: str | None = None ) -> list[Entry]: """ Find similar items in the collection by a given ID. Args: id (str): ID to search by number (int, optional): Number of similar items to return prefix: (str, optional): Filter results to IDs with this prefix Returns: list: List of Entry objects """ import llm matches = list( self.db["embeddings"].rows_where( "collection_id = ? and id = ?", (self.id, id) ) ) if not matches: raise self.DoesNotExist("ID not found") embedding = matches[0]["embedding"] comparison_vector = llm.decode(embedding) return self.similar_by_vector( comparison_vector, number, skip_id=id, prefix=prefix ) def similar( self, value: str | bytes, number: int = 10, prefix: str | None = None ) -> list[Entry]: """ Find similar items in the collection by a given value. Args: value (str or bytes): value to search by number (int, optional): Number of similar items to return prefix: (str, optional): Filter results to IDs with this prefix Returns: list: List of Entry objects """ comparison_vector = self.model().embed(value) return self.similar_by_vector(comparison_vector, number, prefix=prefix) @classmethod def exists(cls, db: Database, name: str) -> bool: """ Does this collection exist in the database? Args: name (str): Name of the collection """ rows = list(db["collections"].rows_where("name = ?", [name])) return bool(rows) def delete(self): """ Delete the collection and its embeddings from the database """ with self.db.atomic(): self.db.execute("delete from embeddings where collection_id = ?", [self.id]) self.db.execute("delete from collections where id = ?", [self.id]) @staticmethod def content_hash(input: str | bytes) -> bytes: "Hash content for deduplication. Override to change hashing behavior." if isinstance(input, str): input = input.encode("utf8") return hashlib.md5(input).digest() ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/embeddings_migrations.py0000644000175100017510000000462315234420023017363 0ustar00runnerrunnerimport hashlib import time from sqlite_utils import Migrations embeddings_migrations = Migrations("llm.embeddings") @embeddings_migrations() def m001_create_tables(db): db["collections"].create({"id": int, "name": str, "model": str}, pk="id") db["collections"].create_index(["name"], unique=True) db["embeddings"].create( { "collection_id": int, "id": str, "embedding": bytes, "content": str, "metadata": str, }, pk=("collection_id", "id"), ) @embeddings_migrations() def m002_foreign_key(db): db["embeddings"].add_foreign_key("collection_id", "collections", "id") @embeddings_migrations() def m003_add_updated(db): db["embeddings"].add_column("updated", int) # Pretty-print the schema db["embeddings"].transform() # Assume anything existing was last updated right now db.execute( "update embeddings set updated = ? where updated is null", [int(time.time())] ) @embeddings_migrations() def m004_store_content_hash(db): db["embeddings"].add_column("content_hash", bytes) db["embeddings"].transform( column_order=( "collection_id", "id", "embedding", "content", "content_hash", "metadata", "updated", ) ) # Register functions manually so we can de-register later def md5(text): return hashlib.md5(text.encode("utf8")).digest() def random_md5(): return hashlib.md5(str(time.time()).encode("utf8")).digest() db.conn.create_function("temp_md5", 1, md5) db.conn.create_function("temp_random_md5", 0, random_md5) with db.atomic(): db.execute(""" update embeddings set content_hash = temp_md5(content) where content is not null """) db.execute(""" update embeddings set content_hash = temp_random_md5() where content is null """) db["embeddings"].create_index(["content_hash"]) # De-register functions db.conn.create_function("temp_md5", 1, None) db.conn.create_function("temp_random_md5", 0, None) @embeddings_migrations() def m005_add_content_blob(db): db["embeddings"].add_column("content_blob", bytes) db["embeddings"].transform( column_order=("collection_id", "id", "embedding", "content", "content_blob") ) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/errors.py0000644000175100017510000000030415234420023014332 0ustar00runnerrunnerclass ModelError(Exception): "Models can raise this error, which will be displayed to the user" class NeedsKeyException(ModelError): "Model needs an API key which has not been provided" ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/hookspecs.py0000644000175100017510000000146415234420023015024 0ustar00runnerrunnerfrom pluggy import HookimplMarker, HookspecMarker hookspec = HookspecMarker("llm") hookimpl = HookimplMarker("llm") @hookspec def register_commands(cli): """Register additional CLI commands, e.g. 'llm mycommand ...'""" @hookspec def register_models(register, model_aliases): "Register additional model instances representing LLM models that can be called" @hookspec def register_embedding_models(register): "Register additional model instances that can be used for embedding" @hookspec def register_template_loaders(register): "Register additional template loaders with prefixes" @hookspec def register_fragment_loaders(register): "Register additional fragment loaders with prefixes" @hookspec def register_tools(register): "Register functions that can be used as tools by the LLMs" ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/logs.py0000644000175100017510000021255115234420023013773 0ustar00runnerrunner"""Content-addressed storage for conversation message trees. A conversation is a parent-linked chain of :class:`llm.Message` objects. Each message is identified by a hash over its own canonical content plus its parent's hash, so two conversations that share a prefix share the rows that store it. Forking a conversation, or re-sending a history from a client that holds the conversation state itself, both write only the messages that are genuinely new. The identity of a message is its *resolved* content, but storage is by reference: text that borrows from a fragment stores the fragment's id in place of a copy, and attachments store an id into the existing content-addressed ``attachments`` table. Ask a hundred questions about a novel and the novel is stored once. Reading resolves the references again, so the hash always covers the content as the model saw it - ``LogStore.verify()`` re-derives it to prove that stays true. """ import datetime import hashlib import json from typing import Any from condense_json import UncondenseError, condense_json, uncondense_json from .migrations import migrate from .models import Attachment, ServerSideTool, _conversation_name from .parts import ( AttachmentPart, Message, Part, ReasoningPart, TextPart, ToolCallPart, ToolResultPart, ) from .utils import ( ensure_fragment, ensure_tool, ensure_tool_instance, make_schema_id, monotonic_ulid, ) __all__ = [ "HASH_PREFIX", "LogStore", "canonical_json", "content_hash", "message_hash", ] # Hashes are tagged with the algorithm that produced them so a future # change to the canonical form or the digest is detectable rather than # silently splitting the dedup space into two incompatible halves. HASH_PREFIX = "b2:" _DIGEST_SIZE = 16 def canonical_json(obj: Any) -> str: """Serialize to the canonical JSON form used for hashing. Keys sorted, no insignificant whitespace, non-ASCII left as-is. This form is part of the documented contract: changing it changes every hash. """ return json.dumps( obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False, ) def content_hash(obj: Any) -> str: "Tagged hash of the canonical JSON form of ``obj``." canonical = canonical_json(obj) digest = hashlib.blake2b( canonical.encode("utf-8"), digest_size=_DIGEST_SIZE ).hexdigest() return f"{HASH_PREFIX}{digest}" def _canonical_attachment(attachment) -> dict: """The hashed form of an attachment: content identity plus the model-visible media type. The content hash is recomputed from the actual bytes every time, never taken from Attachment.id()'s cache - that is what lets verify() notice that a path-backed file changed after logging. Path-backed attachments keep no copy of their bytes in the store (by design: logs.db does not swallow large media), so their fidelity depends on the file staying put; a changed or deleted file shows up as a broken hash rather than passing silently. The type participates because the model sees it: identical bytes sent as image/png and as text/plain are different requests. URL attachments hash the URL itself - the log records which URL was sent, not whatever it served that day. """ if attachment.content: content_id = hashlib.sha256(attachment.content).hexdigest() elif attachment.path: try: with open(attachment.path, "rb") as fp: content_id = hashlib.sha256(fp.read()).hexdigest() except OSError: content_id = f"missing:{attachment.path}" else: content_id = hashlib.sha256( json.dumps({"url": attachment.url}).encode("utf-8") ).hexdigest() try: type_ = attachment.resolve_type() except OSError: # A deleted path-backed file - the content hash above already # carries the missing marker type_ = attachment.type return {"id": content_id, "type": type_} def message_hash(message: Message, parent_hash: str | None) -> str: """Identity of ``message`` when reached via ``parent_hash``. The parent participates, so the same content at a different point in a conversation is a different node. That is what makes a shared prefix collapse to shared rows without any explicit comparison. Attachments are hashed by content id, via _canonical_attachment. """ d: Any = message.to_dict() for part, part_dict in zip(message.parts, d["parts"]): attachment = getattr(part, "attachment", None) if attachment is not None: part_dict["attachment"] = _canonical_attachment(attachment) attachments = getattr(part, "attachments", None) if attachments: part_dict["attachments"] = [_canonical_attachment(a) for a in attachments] return content_hash({"parent": parent_hash, "message": d}) class LogStore: """Read and write conversation history in a SQLite database. Wraps a ``sqlite_utils.Database`` and applies any outstanding migrations on construction, so a fresh database and an existing one are handled the same way:: store = LogStore(sqlite_utils.Database("logs.db")) """ def __init__(self, db): self.db = db migrate(db) # -- writing ------------------------------------------------------- def ensure_chain( self, messages, parent: str | None = None, fragments=None, ) -> str | None: """Store ``messages`` as a chain and return the hash of the tip. Messages already present are left alone, so a caller that re-sends a whole conversation - a client holding the state itself, or a fork of an existing thread - writes only the messages that are new. Passing ``parent`` appends to an existing chain instead of starting a new one. ``fragments`` is an optional list of fragment contents that may appear inside these messages. Any that do are stored as a reference rather than a copy, which is the point of the fragments feature: ask a hundred questions about a novel and the novel is stored once. It never affects the hashes - identity is always the resolved text. """ fragment_map = self._fragment_map(fragments) tip = parent for message in messages: tip = self._ensure_message(message, tip, fragment_map) return tip def _fragment_map(self, fragments) -> dict[str, int]: "Map fragment content to its id, registering any that are new." if not fragments: return {} return { str(fragment): ensure_fragment(self.db, fragment) for fragment in fragments if str(fragment) } def _ensure_message( self, message: Message, parent_hash: str | None, fragment_map: dict[str, int], ) -> str: hash = message_hash(message, parent_hash) if self.db["messages"].count_where("hash = ?", [hash]): # Already stored - and because the hash covers the parent, # everything below it is stored too. return hash with self.db.atomic(): # Another writer can store the same hash between the check # above and this insert. Insert-or-ignore settles who won, # and only the winner writes the parts. cursor = self.db.execute( "insert or ignore into messages" " (hash, parent_hash, role, provider_metadata)" " values (?, ?, ?, ?)", [hash, parent_hash, message.role, _dump(message.provider_metadata)], ) if cursor.rowcount: for position, part in enumerate(message.parts): self._write_part(hash, position, part, fragment_map) return hash def _write_part( self, message_hash_: str, position: int, part, fragment_map: dict[str, int], ) -> None: payload = part.to_dict() # The type key is redundant with the type column; readers put it # back from there. part_type = payload.pop("type") attachments = _attachments_of(part) # Large content out of the payload and into the tables that # already store it once: fragments for text, attachments for # bytes. Both are resolved again on the way back out. used_fragments = _encode_text_refs(payload, fragment_map) if attachments: attachment_ids = [ ensure_attachment(self.db, attachment) for attachment in attachments ] _encode_attachment_refs(payload, attachment_ids, part_type) else: attachment_ids = [] # Pure literal text lives in its own column - raw, unescaped, # never parsed - so prose reads as prose in SQL. Text that # borrows fragments stays structured in the payload as text_ref. text = None if part_type in ("text", "reasoning") and "text" in payload: text = payload.pop("text") part_id = ( self.db["parts"] .insert( { "message_hash": message_hash_, "position": position, "type": part_type, "tool_name": payload.get("name"), "text": text, # Plain dumps, not canonical_json: sorting keys is # for hashing. Storage keeps the order the model # produced, so tool call arguments read back as # they were written. NULL when the text column # carries the whole part. "payload": json.dumps(payload) if payload else None, } ) .last_pk ) for order, attachment_id in enumerate(attachment_ids): self.db["part_attachments"].insert( { "part_id": part_id, "attachment_id": attachment_id, "order": order, } ) for order, fragment_id in enumerate(used_fragments): self.db["part_fragments"].insert( { "part_id": part_id, "fragment_id": fragment_id, "order": order, } ) # -- reading ------------------------------------------------------- def load_chain(self, tip: str | None) -> list[Message]: """Return the full chain ending at ``tip``, oldest message first. Raises ``KeyError`` if ``tip`` is not in the store. """ if tip is None: return [] rows = [] hash: str | None = tip while hash is not None: found = list(self.db.query("select * from messages where hash = ?", [hash])) if not found: raise KeyError(hash) rows.append(found[0]) hash = found[0]["parent_hash"] rows.reverse() parts_by_message = self._load_parts([row["hash"] for row in rows]) return [ Message( role=row["role"], parts=parts_by_message.get(row["hash"], []), provider_metadata=_load(row["provider_metadata"]), ) for row in rows ] def _load_parts(self, message_hashes: list[str]) -> dict[str, list[Any]]: if not message_hashes: return {} placeholders = ",".join("?" * len(message_hashes)) part_rows = list( self.db.query( f""" select * from parts where message_hash in ({placeholders}) order by message_hash, position """, message_hashes, ) ) # Rebuild each part's full dict from its columns: type from the # type column, literal text from the text column, everything # else from the JSON payload. payloads: list[Any] = [] for row in part_rows: payload = json.loads(row["payload"]) if row["payload"] else {} payload["type"] = row["type"] if row["text"] is not None: payload["text"] = row["text"] payloads.append(payload) # Resolve the references put in on the way in. Both lookups are # batched across the whole chain rather than done per part. fragments = self._load_fragments(payloads) attachments = self._load_attachments(payloads) out: dict[str, list[Any]] = {} for row, payload in zip(part_rows, payloads): _decode_text_refs(payload, fragments) # Strip the attachment references before rebuilding, then # hang the resolved objects back on. ids = _attachment_ids(payload) payload.pop("attachment", None) payload.pop("attachments", None) part = Part.from_dict(payload) _resolve_attachments(part, ids, attachments) out.setdefault(row["message_hash"], []).append(part) return out def _load_fragments(self, payloads: list[dict]) -> dict[int, str]: ids = sorted({id for payload in payloads for id in _fragment_ids(payload)}) if not ids: return {} placeholders = ",".join("?" * len(ids)) return { row["id"]: row["content"] for row in self.db.query( f"select id, content from fragments where id in ({placeholders})", ids, ) } def _load_attachments(self, payloads: list[dict]) -> dict[str, Any]: ids = sorted({id for payload in payloads for id in _attachment_ids(payload)}) if not ids: return {} placeholders = ",".join("?" * len(ids)) return { row["id"]: Attachment.from_row(row) for row in self.db.query( f"select * from attachments where id in ({placeholders})", ids ) } # -- threads ------------------------------------------------------- def create_thread( self, name: str | None = None, tip: str | None = None, forked_from: str | None = None, id: str | None = None, ) -> str: "Create a named pointer at a message and return its id." thread_id = id or str(monotonic_ulid()).lower() self.db["threads"].insert( { "id": thread_id, "name": name, "tip_message_hash": tip, "forked_from": forked_from, "datetime_utc": _now(), } ) return thread_id def ensure_thread(self, thread_id: str, name: str | None = None) -> str: """Return the thread with this id, creating it if it is new. Threads created from a conversation reuse the conversation's id, so the two identifier spaces line up while both sets of tables are being written. """ if not self.db["threads"].count_where("id = ?", [thread_id]): self.create_thread(name=name, id=thread_id) return thread_id def fork( self, message_hash_: str, name: str | None = None, forked_from: str | None = None, ) -> str: """Start a new thread from an existing message. Nothing is copied - the new thread points at a message that is already stored, so its whole history is shared with the thread it came from until the two diverge. """ if not self.db["messages"].count_where("hash = ?", [message_hash_]): raise KeyError(message_hash_) return self.create_thread(name=name, tip=message_hash_, forked_from=forked_from) def thread_tip(self, thread_id: str) -> str | None: "The message a thread currently points at." rows = list(self.db.query("select * from threads where id = ?", [thread_id])) if not rows: raise KeyError(thread_id) return rows[0]["tip_message_hash"] def thread_messages(self, thread_id: str) -> list[Message]: "The full history of a thread, oldest message first." return self.load_chain(self.thread_tip(thread_id)) def append(self, thread_id: str, messages) -> str | None: "Add messages to the end of a thread and return the new tip." tip = self.ensure_chain(messages, parent=self.thread_tip(thread_id)) self.db["threads"].update(thread_id, {"tip_message_hash": tip}) return tip # -- turns --------------------------------------------------------- def log(self, response, thread_id: str | None = None) -> str: """Record a completed response. The input chain and the response's own output are stored as messages; everything that is specific to this particular call - timings, usage, which model answered - goes on the turn, because message rows are shared and so cannot carry provenance. """ with self.db.atomic(): return self._log_in_transaction(response, thread_id) def _log_in_transaction(self, response, thread_id: str | None) -> str: if thread_id is None: conversation = getattr(response, "conversation", None) # A response logged outside any conversation still gets a # thread of its own, so `llm -c` and `llm logs` can always # find it - the same guarantee the conversations table used # to provide. thread_id = self.ensure_thread( conversation.id if conversation else str(monotonic_ulid()).lower(), name=_conversation_name( response.prompt.prompt or response.prompt.system or "" ), ) prompt_fragments = list(response.prompt.fragments or []) system_fragments = list(response.prompt.system_fragments or []) parent = self.ensure_chain( response.prompt.messages, fragments=prompt_fragments + system_fragments, ) # _messages_now() rather than messages(), which is a coroutine on # AsyncResponse. own_messages = response._messages_now() tip = self.ensure_chain(own_messages, parent=parent) schema_id = None if response.prompt.schema: schema_id, schema_json = make_schema_id(response.prompt.schema) self.db["schemas"].insert( {"id": schema_id, "content": schema_json}, ignore=True ) turn_id = response.id or str(monotonic_ulid()).lower() self.db["turns"].insert( { "id": turn_id, "thread_id": thread_id, "parent_message_hash": parent, "tip_message_hash": tip, "model": response.model.model_id, "resolved_model": response.resolved_model, "options_json": _dump( { key: value for key, value in dict(response.prompt.options).items() if value is not None } ), "schema_id": schema_id, "input_tokens": response.input_tokens, "output_tokens": response.output_tokens, "token_details": _dump(response.token_details), "duration_ms": response.duration_ms(), "datetime_utc": response.datetime_utc(), "response_json": condense_payload( getattr(response, "response_json", None), own_messages, [(tool.name, tool.description) for tool in response.prompt.tools], schema=response.prompt.schema, model_replacements=getattr( response.model, "json_replacements", None ), ), }, replace=True, ) for tool in response.prompt.tools: # Server-side tools are configured instances themselves. A # toolbox-derived tool instead has an implementation method # bound to its configured instance. Record either kind as a # reference into the shared tool_instances table. instance: Any | None if isinstance(tool, ServerSideTool): instance = tool instance_name = tool.__class__.__name__ else: instance = getattr(tool.implementation, "__self__", None) instance_name = tool.name.split("_")[0] config = getattr(instance, "_config", None) self.db["turn_tools"].insert( { "turn_id": turn_id, "tool_id": ensure_tool(self.db, tool), "instance_id": ( ensure_tool_instance( self.db, instance_name, tool.plugin, json.dumps(config), ) if config is not None else None ), }, replace=True, ) # Which fragments this call was given - provenance, so it belongs # on the turn rather than on the shared message rows. This is what # answers "show me everything that used fragment X". for kind, fragments in ( ("prompt", prompt_fragments), ("system", system_fragments), ): for order, fragment in enumerate(fragments): self.db["turn_fragments"].insert( { "turn_id": turn_id, "fragment_id": ensure_fragment(self.db, fragment), "order": order, "kind": kind, }, replace=True, ) # Which configured toolbox instance served each tool call. This # is local execution provenance, so it lives outside the hashed # message tree, keyed by the tool_call_id both worlds share. for tool_result in response.prompt.tool_results: # instance is annotated as Toolbox but a tool built from a # bound method can carry an arbitrary __self__ here, so ask # for the config rather than trusting the type. config = getattr(tool_result.instance, "_config", None) if config is None or not tool_result.tool_call_id: continue self.db["tool_instantiations"].insert( { "turn_id": turn_id, "tool_call_id": tool_result.tool_call_id, "instance_id": ensure_tool_instance( self.db, tool_result.name.split("_")[0], next( ( tool.plugin for tool in response.prompt.tools if tool.name == tool_result.name ), None, ), json.dumps(config), ), }, replace=True, ) # Refresh this turn's search row. Delete-then-derive rather than # replace, so a re-logged turn with a different tip converges on # what the turn now says. Derived in SQL from the stored # payloads, not from load_chain - resolving references would put # fragment content back into the searchable text. self.db["turn_search"].delete_where("turn_id = ?", [turn_id]) self.db.execute(TURN_SEARCH_INSERT_SQL, {"turn_id": turn_id}) if thread_id is not None: self.db["threads"].update(thread_id, {"tip_message_hash": tip}) return turn_id def turn_response_json(self, turn_id: str) -> Any: """The raw provider payload recorded for a turn, resolved. Returns ``None`` when the turn is unknown or recorded no payload. Raises ``condense_json.UncondenseError`` when the payload references message content that no longer resolves - the payload was recorded but its context is gone. """ row = next( iter( self.db.query( "select turns.model, turns.parent_message_hash," " turns.tip_message_hash, turns.response_json," " schemas.content as schema_json" " from turns" " left join schemas on turns.schema_id = schemas.id" " where turns.id = ?", [turn_id], ) ), None, ) if row is None or row["response_json"] is None: return None inputs = self.load_chain(row["parent_message_hash"]) outputs = self.load_chain(row["tip_message_hash"])[len(inputs) :] return resolve_payload( row["response_json"], outputs, self._turn_tool_pairs(turn_id), schema=_load(row["schema_json"]), model_replacements=_model_json_replacements(row["model"]), ) def _turn_tool_pairs(self, turn_id: str) -> list[tuple[str, str]]: return [ (row["name"], row["description"]) for row in self.db.query(TURN_TOOLS_SQL, [turn_id]) ] # -- verification -------------------------------------------------- def verify(self) -> list[str]: """Re-hash every stored message and return those that disagree. Reads resolve references - fragment text and attachment bytes are stitched back in - so a bug there, or a fragment deleted out from under a message, produces a chain that differs from the one that was hashed. Nothing else would notice: the wrong text would just be silently sent to the model. Re-deriving the hash from what comes back out catches the whole class. An empty list means every message on disk still resolves to the content its hash was taken over. """ broken = [] for row in self.db.query("select hash, parent_hash from messages"): parts = self._load_parts([row["hash"]]).get(row["hash"], []) message_row = next( iter( self.db.query( "select * from messages where hash = ?", [row["hash"]] ) ) ) message = Message( role=message_row["role"], parts=parts, provider_metadata=_load(message_row["provider_metadata"]), ) if message_hash(message, row["parent_hash"]) != row["hash"]: broken.append(row["hash"]) return broken # -- pending work -------------------------------------------------- def pending_tool_calls(self, tip: str | None) -> list[Any]: """Tool calls at the tip of a chain that have no result yet. A chain ending in tool calls with nothing after them is a paused conversation waiting to be resumed - it needs no separate record. """ chain = self.load_chain(tip) if not chain: return [] return [ part for part in chain[-1].parts if isinstance(part, ToolCallPart) and not part.server_executed ] def ensure_attachment(db, attachment) -> str: "Store an attachment, returning its content-addressed id." attachment_id = attachment.id() db["attachments"].insert( { "id": attachment_id, "type": attachment.resolve_type(), "path": attachment.path, "url": attachment.url, "content": attachment.content, }, replace=True, ) return attachment_id # -- reference encoding ------------------------------------------------ # # A stored payload is Part.to_dict() with large content swapped for a # reference: fragment ids in place of text, attachment ids in place of # bytes. Resolving it reproduces the wire form exactly, which is what # makes it safe for the hash to be taken over the resolved content and # never over what is on disk. def _attachments_of(part) -> list[Any]: "The Attachment objects a part carries, in order." if isinstance(part, ToolResultPart): return list(part.attachments) if isinstance(part, AttachmentPart) and part.attachment is not None: return [part.attachment] return [] def _encode_text_refs(payload: dict, fragment_map: dict[str, int]) -> list[int]: """Replace ``text`` with a ``text_ref`` list of fragments and literals. Returns the fragment ids used, in order. Nothing is replaced unless a fragment actually occurs in the text, so a part that borrows no fragments keeps its plain ``text`` key. """ text = payload.get("text") if not text or not fragment_map: return [] pieces: list[dict] = [] used: list[int] = [] remaining = text while remaining: # Earliest occurrence wins; on a tie the longer fragment does, so # a fragment that is a prefix of another cannot mask it. best: tuple[int, str] | None = None for content in fragment_map: index = remaining.find(content) if index == -1: continue if best is None or (index, -len(content)) < (best[0], -len(best[1])): best = (index, content) if best is None: pieces.append({"literal": remaining}) break index, content = best if index: pieces.append({"literal": remaining[:index]}) pieces.append({"fragment": fragment_map[content]}) used.append(fragment_map[content]) remaining = remaining[index + len(content) :] if not used: return [] del payload["text"] payload["text_ref"] = pieces return used def _decode_text_refs(payload: dict, fragments: dict[int, str]) -> None: "Reverse of _encode_text_refs, restoring the exact original text." pieces = payload.pop("text_ref", None) if pieces is None: return payload["text"] = "".join( ( piece["literal"] if "literal" in piece else fragments.get(piece["fragment"], "") ) for piece in pieces ) def _fragment_ids(payload: dict) -> list[int]: return [ piece["fragment"] for piece in payload.get("text_ref") or [] if "fragment" in piece ] def _encode_attachment_refs( payload: dict, attachment_ids: list[str], part_type: str ) -> None: "Replace inline attachment dicts with their content-addressed ids." if part_type == "attachment": payload["attachment"] = {"id": attachment_ids[0]} elif part_type == "tool_result": payload["attachments"] = [{"id": id} for id in attachment_ids] def _resolve_attachments(part, ids: list[str], attachments: dict[str, Any]) -> None: """Hang the resolved Attachment objects back on a part. Done after ``Part.from_dict`` rather than by putting them back into the payload, so the bytes are never round-tripped through base64 and the objects keep the content-addressed id they were stored under. """ if not ids: return if isinstance(part, AttachmentPart): part.attachment = attachments[ids[0]] elif isinstance(part, ToolResultPart): part.attachments = [attachments[id] for id in ids] def _attachment_ids(payload: dict) -> list[str]: if payload["type"] == "attachment" and "attachment" in payload: return [payload["attachment"]["id"]] if payload["type"] == "tool_result": return [ref["id"] for ref in payload.get("attachments") or []] return [] def _dump(value: dict | None) -> str | None: return json.dumps(value) if value else None def _load(value: str | None) -> dict | None: return json.loads(value) if value else None def _now() -> str: return str(datetime.datetime.now(datetime.timezone.utc)) # -- condensed provider payloads ---------------------------------------- # # The raw response.json() payload mostly duplicates content the store # already holds: the response text, reasoning summaries and their # encrypted blobs, long tool arguments, and the tool definitions the # provider echoes back on every call. The turn stores it condensed # instead - strings that already live in the turn's own messages or its # tools are swapped for {"$": key} references (condense-json), leaving # roughly the provider envelope: ids, usage, fingerprints, settings # echoes. The replacement dict is never stored; it is rebuilt on the way # out from the chain segment (hash-frozen) and the turn_tools join # (per-turn provenance), so the same walk over the same rows produces # the same dict on both sides of the round trip. # Below this a {"$": key} marker costs about as much as the string it # replaces. _CONDENSE_MIN_LENGTH = 64 # The tools a turn was given, as the (name, description) pairs # _payload_replacements expects. TURN_TOOLS_SQL = """ select tools.name, tools.description from turn_tools join tools on tools.id = turn_tools.tool_id where turn_tools.turn_id = ? """ def _payload_replacements( messages, tools=(), schema=None, model_replacements=None ) -> dict[str, Any]: """Replacement values for condensing a turn's provider payload. ``messages`` is the turn's own contribution - the chain segment between its parent and tip. Keys are structural (message offset, part position, field path) so the identical dict can be rebuilt from the stored segment at read time. ``tools`` is the turn's tools as (name, description) pairs - ``response.prompt.tools`` on the way in, the ``turn_tools`` join on the way out. Descriptions are keyed by tool name; a name carrying two different descriptions in one turn is dropped, the same order-independent verdict from either side's view of the pairs. (Parameter schemas are deliberately not offered: providers echo a transformed schema - OpenAI strict mode adds keys - so the stored form would not match even structurally.) ``schema`` is the turn's JSON schema dict, when one was used - the provider echoes it back in the payload (OpenAI ``text.format``) and the schemas table already stores it once. It is offered as a structural value, so the echo's key order does not matter. ``model_replacements`` is the model class's ``json_replacements`` dictionary of common boilerplate - the zstd-custom-dictionary idea: payload fragments the plugin author knows recur in every reply, keyed here under an ``m.`` prefix so they can never collide with the derived keys. These resolve by looking the model up again at read time, so plugins must treat their ``json_replacements`` as append-only: removing or changing an entry breaks every payload already stored against it. Values are strings (matched as substrings) and dicts or lists (matched as whole subtrees by condense-json's structural equality). Container values inside provider_metadata are offered as well as their leaf strings - outermost match wins, so a payload that embeds a whole metadata object (a reasoning ``summary`` list, say) condenses to one reference instead of one per string. """ replacements: dict[str, Any] = {} def add(key: str, value: Any) -> None: if isinstance(value, str) and len(value) >= _CONDENSE_MIN_LENGTH: replacements[key] = value def add_value(key: str, value: Any) -> None: if isinstance(value, (dict, list)) and value: # The same length bar as strings, applied to the canonical # form, so a reference is always a clear win. try: size = len(json.dumps(value, separators=(",", ":"))) except (TypeError, ValueError): return if size >= _CONDENSE_MIN_LENGTH: replacements[key] = value def walk(prefix: str, obj: Any) -> None: if isinstance(obj, dict): add_value(prefix, obj) for key, value in obj.items(): walk(f"{prefix}.{key}", value) elif isinstance(obj, list): add_value(prefix, obj) for index, value in enumerate(obj): walk(f"{prefix}.{index}", value) else: add(prefix, obj) for mi, message in enumerate(messages): message_dict = message.to_dict() for pi, part in enumerate(message_dict.get("parts", [])): base = f"{mi}.{pi}" add(f"{base}.text", part.get("text")) add(f"{base}.output", part.get("output")) arguments = part.get("arguments") if arguments: # Providers that carry tool arguments as a JSON-encoded # string need a byte-exact serialization - OpenAI uses # the compact form, so offer both. Providers that embed # them as an object (Anthropic, Gemini) match the dict # itself structurally. compact = json.dumps(arguments, separators=(",", ":")) spaced = json.dumps(arguments) add(f"{base}.args", compact) if spaced != compact: add(f"{base}.args2", spaced) add_value(f"{base}.argsv", arguments) walk(f"{base}.pm", part.get("provider_metadata") or {}) walk(f"{mi}.pm", message_dict.get("provider_metadata") or {}) descriptions: dict[str, str] = {} conflicting = set() for name, description in tools: if ( not name or not isinstance(description, str) or len(description) < _CONDENSE_MIN_LENGTH ): continue if descriptions.get(name, description) != description: conflicting.add(name) continue descriptions[name] = description for name, description in descriptions.items(): if name not in conflicting: replacements[f"tool.{name}.description"] = description add_value("schema", schema) # Model-declared boilerplate passes through as curated: the plugin # author chose these entries, so no length threshold applies. for key, value in (model_replacements or {}).items(): replacements[f"m.{key}"] = value return replacements def condense_payload( payload: Any, messages, tools=(), schema=None, model_replacements=None ) -> str | None: "JSON text of ``payload`` condensed against the turn's stored content." if payload is None: return None return json.dumps( condense_json( payload, _payload_replacements(messages, tools, schema, model_replacements), ) ) def resolve_payload( condensed: str | None, messages, tools=(), schema=None, model_replacements=None ) -> Any: """Reverse of :func:`condense_payload`. Raises ``condense_json.UncondenseError`` when the stored payload references content the segment no longer produces. """ if condensed is None: return None return uncondense_json( json.loads(condensed), _payload_replacements(messages, tools, schema, model_replacements), ) def _model_json_replacements(model_id: str | None): """The ``json_replacements`` boilerplate dictionary for a model id. Resolved through the registry at read time, so payloads condensed against a model's dictionary need that model's plugin installed to resolve again. An unknown model returns None; any markers that depended on it surface as UncondenseError from resolve_payload. """ if not model_id: return None from llm import UnknownModelError, get_model try: model = get_model(model_id) except UnknownModelError: return None return getattr(model, "json_replacements", None) # -- llm logs support --------------------------------------------------- # # Rows shaped like the ones the older `responses` query produced, so the # existing rendering in llm.cli works unchanged, but derived entirely # from the content-addressed tables. LOG_ROWS_SQL = """ select turns.id, turns.model, turns.resolved_model, turns.options_json, turns.thread_id as conversation_id, turns.duration_ms, turns.datetime_utc, turns.input_tokens, turns.output_tokens, turns.token_details, turns.parent_message_hash, turns.tip_message_hash, turns.response_json, threads.name as conversation_name, turns.model as conversation_model, schemas.content as schema_json{rank_select} from turns left join threads on turns.thread_id = threads.id left join schemas on turns.schema_id = schemas.id{join} {where} order by {order_by}{limit} """ # Literal text of one parts row: the text column when the part's text # is pure literal, otherwise the literal segments of a text_ref payload # - fragment content is deliberately not searchable. TURN_SEARCH_LITERAL = """coalesce( parts.text, (select group_concat(json_extract(je.value, '$.literal'), '') from json_each(parts.payload, '$.text_ref') je where json_extract(je.value, '$.literal') is not null) )""" # Derives the searchable prompt and response text for turns. Serves both # the migration backfill (turn_filter="") and the per-turn refresh in # LogStore.log (turn_filter="and turns.id = :turn_id" - the slot appears # in three places so the filtered form touches only that turn's chain). TURN_SEARCH_INSERT_SQL = """ with recursive output_messages(turn_id, hash) as ( select turns.id, turns.tip_message_hash from turns where turns.tip_message_hash is not null and (turns.parent_message_hash is null or turns.tip_message_hash != turns.parent_message_hash) and turns.id = :turn_id union all select om.turn_id, messages.parent_hash from output_messages om join messages on messages.hash = om.hash join turns on turns.id = om.turn_id where messages.parent_hash is not null and (turns.parent_message_hash is null or messages.parent_hash != turns.parent_message_hash) ), prompt_text as ( select turns.id as turn_id, (select group_concat({LITERAL}, '') from parts where parts.message_hash = turns.parent_message_hash and parts.type = 'text' order by parts.position) as text from turns join messages on messages.hash = turns.parent_message_hash where messages.role = 'user' and turns.id = :turn_id ), response_text as ( select om.turn_id, group_concat(part_text.text, '') as text from output_messages om join messages on messages.hash = om.hash and messages.role = 'assistant' join ( select parts.message_hash, parts.position, {LITERAL} as text from parts where parts.type = 'text' ) part_text on part_text.message_hash = om.hash group by om.turn_id ) insert into turn_search (turn_id, prompt, response) select turns.id, coalesce(prompt_text.text, ''), coalesce(response_text.text, '') from turns left join prompt_text on prompt_text.turn_id = turns.id left join response_text on response_text.turn_id = turns.id where (coalesce(prompt_text.text, '') != '' or coalesce(response_text.text, '') != '') and turns.id = :turn_id """.replace("{LITERAL}", TURN_SEARCH_LITERAL) # Relevance ranking for -q. bm25 scores are negative-better, ascending # order is best-first. The prompt column is weighted well above the # response: what you typed is a stronger signal of what a turn is about # than what the model said back. TURN_SEARCH_RANK = "bm25(turn_search_fts, 10.0, 1.0)" LEGACY_SEARCH_RANK = "bm25(responses_fts, 10.0, 1.0)" def _text_of(parts, kind) -> str: "Concatenated text of every part of ``kind`` in order." return "".join(part.text for part in parts if isinstance(part, kind) and part.text) class _LogRowBuilder: """Turns a turn row into the shape `llm logs` renders. A turn's prompt is the last message it was given and its response is whatever it appended, so both are derived from the parent/tip pair rather than stored a second time. The chain up to the parent is a prefix of the chain up to the tip, so splitting them is a matter of length. """ def __init__(self, store: "LogStore"): self.store = store def build(self, row: dict) -> dict: inputs = self.store.load_chain(row["parent_message_hash"]) outputs = self.store.load_chain(row["tip_message_hash"])[len(inputs) :] # The turn's own input is the trailing run of user and tool # messages: everything after the last assistant (or system) # message belongs to this turn, because a turn's new input # never contains an assistant message. A turn carrying both # tool results and a fresh user prompt therefore keeps both. boundary = len(inputs) while boundary and inputs[boundary - 1].role in ("user", "tool"): boundary -= 1 input_messages = inputs[boundary:] prompt_parts = [ part for message in input_messages if message.role == "user" for part in message.parts ] input_parts = [part for message in input_messages for part in message.parts] system_parts = inputs[0].parts if inputs and inputs[0].role == "system" else [] out_parts = [part for message in outputs for part in message.parts] built = { key: row[key] for key in ( "id", "model", "resolved_model", "options_json", "conversation_id", "duration_ms", "datetime_utc", "input_tokens", "output_tokens", "token_details", "conversation_name", "conversation_model", "schema_json", ) } if "_search_rank" in row: built["_search_rank"] = row["_search_rank"] built.update( { # The turn stores null when no options were set; the # responses table always recorded "{}". "options_json": row["options_json"] or "{}", "prompt": _text_of(prompt_parts, TextPart), # None rather than "" when there was no system # message, matching what was recorded before. "system": _text_of(system_parts, TextPart) or None, "response": _text_of(out_parts, TextPart), "reasoning": _text_of(out_parts, ReasoningPart) or None, # No longer stored: the chain holds the structure. "prompt_json": None, # Stored condensed against the turn's own messages; # resolved here so the row carries the payload as the # provider sent it. A payload whose references no longer # resolve renders as absent rather than failing the # whole listing. "response_json": self._resolve_response_json(row, outputs), "_input_parts": input_parts, "_output_parts": out_parts, # Internal, stripped before rendering - the enrichment # needs them to find the parts rows behind these parts. "_parent_message_hash": row["parent_message_hash"], "_input_message_hashes": self._input_segment_hashes( row["parent_message_hash"] ), "_tip_message_hash": row["tip_message_hash"], } ) return built def _resolve_response_json(self, row: dict, outputs) -> str | None: condensed = row.get("response_json") if not condensed: return None try: return json.dumps( resolve_payload( condensed, outputs, self.store._turn_tool_pairs(row["id"]), schema=_load(row.get("schema_json")), model_replacements=_model_json_replacements(row.get("model")), ) ) except UncondenseError: return None def _input_segment_hashes(self, parent_hash: str | None) -> list[str]: """Hashes of the turn's own input messages - the same trailing user/tool run build() derives, walked directly in the table.""" hashes: list[str] = [] hash_ = parent_hash while hash_: message_row = next( iter( self.store.db.query( "select parent_hash, role from messages where hash = ?", [hash_], ) ), None, ) if message_row is None or message_row["role"] not in ("user", "tool"): break hashes.append(hash_) hash_ = message_row["parent_hash"] hashes.reverse() return hashes def log_rows( store: "LogStore", *, count: int | None = None, model_id: str | None = None, thread_id: str | None = None, fragment_hashes=(), tool_names=(), any_tools: bool = False, schema_id: str | None = None, id_gt: str | None = None, id_gte: str | None = None, ids=(), query: str | None = None, latest: bool = False, ) -> list[dict]: """Rows for `llm logs`, newest first, drawn from the new tables. Sees only conversations with turns - merged_log_rows adds the rows that exist solely in the legacy `responses` table. With ``query`` the rows are the best matches from the turn_search index, most relevant first - or newest first when ``latest`` is also set. """ where: list[str] = [] params: dict[str, Any] = {} if model_id: where.append("(turns.model = :model or turns.resolved_model = :model)") params["model"] = model_id if thread_id: where.append("turns.thread_id = :thread_id") params["thread_id"] = thread_id if id_gt: where.append("turns.id > :id_gt") params["id_gt"] = id_gt if id_gte: where.append("turns.id >= :id_gte") params["id_gte"] = id_gte if schema_id: where.append("turns.schema_id = :schema_id") params["schema_id"] = schema_id if ids: keys = [f"row_id_{index}" for index in range(len(ids))] where.append("turns.id in ({})".format(", ".join(f":{key}" for key in keys))) params.update(dict(zip(keys, ids))) # Fragments come from turn_fragments - what this call was given - # rather than from the message text, so it matches what -f means. for index, fragment_hash in enumerate(fragment_hashes): key = f"fragment_{index}" where.append(f"""turns.id in ( select turn_fragments.turn_id from turn_fragments join fragments on fragments.id = turn_fragments.fragment_id where fragments.hash = :{key} )""") params[key] = fragment_hash # A turn "used" a tool when a tool result was among its inputs, # matching what -T has always meant: the result came back and was # fed to the model. The result sits in the turn's parent message. if any_tools: where.append(_tool_result_clause()) for index, tool_name in enumerate(tool_names): key = f"tool_{index}" where.append(_tool_result_clause(f"and parts.tool_name = :{key}")) params[key] = tool_name rank_select = "" join = "" order_by = "turns.id desc" if query: rank_select = f",\n {TURN_SEARCH_RANK} as _search_rank" join = ( "\njoin turn_search on turn_search.turn_id = turns.id" "\njoin turn_search_fts on turn_search_fts.rowid = turn_search.id" ) where.append("turn_search_fts match :query") params["query"] = query if not latest: order_by = TURN_SEARCH_RANK sql = LOG_ROWS_SQL.format( rank_select=rank_select, join=join, where=("where " + " and ".join(where)) if where else "", order_by=order_by, limit=f" limit {count}" if count else "", ) builder = _LogRowBuilder(store) return [builder.build(row) for row in store.db.query(sql, params)] def _tool_result_clause(extra: str = "") -> str: # A turn's tool results sit either in its parent message directly, # or - when a fresh user prompt followed the results - in the # parent's own parent, one step further up the same input segment. return f"""(turns.parent_message_hash in ( select parts.message_hash from parts where parts.type = 'tool_result' {extra} ) or turns.parent_message_hash in ( select messages.hash from messages join parts on parts.message_hash = messages.parent_hash where messages.role = 'user' and parts.type = 'tool_result' {extra} ))""" def log_row_extras(store: "LogStore", row: dict) -> dict: """Attachments, fragments and tool info for one `llm logs` row. Attachments and tool calls come from the row's parts, which the row builder kept hold of. Fragments come from turn_fragments - what the call was given - so `-f` and the displayed list agree. """ attachments = [ _attachment_summary(part.attachment) for part in row.get("_input_parts", []) if isinstance(part, AttachmentPart) and part.attachment is not None ] # parts.id and tools.id stand in for the row ids the old # tool_calls / tool_results tables exposed, so the JSON shape of # `llm logs` is unchanged. call_ids = _part_ids(store, [row.get("_tip_message_hash")], "tool_call") result_ids = _part_ids(store, row.get("_input_message_hashes") or [], "tool_result") # input_schema is rendered as a dict, so decode it here rather than # handing the caller the raw JSON text out of the column. tools = [ { "id": tool_row["id"], "hash": tool_row["hash"], "name": tool_row["name"], "description": tool_row["description"], "input_schema": json.loads(tool_row["input_schema"] or "{}"), "instance": ( { "name": tool_row["instance_name"], "arguments": tool_row["instance_arguments"], } if tool_row["instance_name"] else None ), } for tool_row in store.db.query( """ select tools.id, tools.hash, tools.name, tools.description, tools.input_schema, tool_instances.name as instance_name, tool_instances.arguments as instance_arguments from tools join turn_tools on turn_tools.tool_id = tools.id left join tool_instances on tool_instances.id = turn_tools.instance_id where turn_tools.turn_id = ? """, [row["id"]], ) ] # Resolved through this turn's own turn_tools rows - definitions # sharing a name can differ between turns, and a global map would # attribute the wrong one. tool_ids = {tool["name"]: tool["id"] for tool in tools} tool_calls = [ { "id": call_ids.get(part.tool_call_id), "tool_id": tool_ids.get(part.name), "name": part.name, "arguments": part.arguments, "tool_call_id": part.tool_call_id, } for part in row.get("_output_parts", []) if isinstance(part, ToolCallPart) ] result_parts = [ part for part in row.get("_input_parts", []) if isinstance(part, ToolResultPart) ] instances = _instances_by_tool_call_id( store, row["id"], [part.tool_call_id for part in result_parts if part.tool_call_id], ) tool_results = [ { "id": result_ids.get(part.tool_call_id), "tool_id": tool_ids.get(part.name), "name": part.name, "output": part.output, "tool_call_id": part.tool_call_id, "exception": part.exception, "instance": instances.get(part.tool_call_id), "attachments": [_attachment_summary(a) for a in part.attachments], } for part in result_parts ] fragments: dict[str, list[dict]] = { "prompt_fragments": [], "system_fragments": [], } for fragment_row in store.db.query( """ select turn_fragments.kind, fragments.hash, fragments.content, (select json_group_array(fragment_aliases.alias) from fragment_aliases where fragment_aliases.fragment_id = fragments.id) as aliases from turn_fragments join fragments on fragments.id = turn_fragments.fragment_id where turn_fragments.turn_id = ? order by turn_fragments."order" """, [row["id"]], ): key = f"{fragment_row['kind']}_fragments" fragments[key].append(dict(fragment_row)) return { "attachments": attachments, "tools": tools, "tool_calls": tool_calls, "tool_results": tool_results, **fragments, } def _attachment_summary(attachment) -> dict: "The attachment shape `llm logs` renders." content = attachment.content or b"" return { "id": attachment.id(), "type": attachment.resolve_type(), "path": attachment.path, "url": attachment.url, "content": bool(content) or None, "content_length": len(content) or None, } def _part_ids(store: "LogStore", message_hashes: list, type: str) -> dict: "Map tool_call_id to the parts row id, across the given messages." message_hashes = [hash_ for hash_ in message_hashes if hash_] if not message_hashes: return {} placeholders = ",".join("?" * len(message_hashes)) return { json.loads(part_row["payload"]).get("tool_call_id"): part_row["id"] for part_row in store.db.query( f"select id, payload from parts" f" where message_hash in ({placeholders}) and type = ?", message_hashes + [type], ) } def _instances_by_tool_call_id( store: "LogStore", turn_id: str, tool_call_ids: list ) -> dict: """Which configured toolbox instance served each call, for display. Scoped to the turn: providers with per-request counters can reuse the same tool_call_id across independent turns. """ if not tool_call_ids: return {} placeholders = ",".join("?" * len(tool_call_ids)) return { row["tool_call_id"]: { "name": row["name"], "plugin": row["plugin"], "arguments": row["arguments"], } for row in store.db.query( f""" select tool_instantiations.tool_call_id, tool_instances.name, tool_instances.plugin, tool_instances.arguments from tool_instantiations join tool_instances on tool_instances.id = tool_instantiations.instance_id where tool_instantiations.turn_id = ? and tool_instantiations.tool_call_id in ({placeholders}) """, [turn_id] + tool_call_ids, ) } # -- legacy rows --------------------------------------------------------- # # History logged by older versions of llm lives only in the `responses` # table. Those rows are merged into `llm logs` output, shaped like the # rows _LogRowBuilder produces. A response whose id also exists in # `turns` is suppressed - that is what a dual-write-era row or a # backfilled conversation looks like, and the turn is the richer record. LEGACY_LOG_ROWS_SQL = """ select responses.id, responses.model, responses.resolved_model, responses.prompt, responses.system, responses.prompt_json, responses.options_json, responses.response, responses.reasoning, responses.response_json, responses.conversation_id, responses.duration_ms, responses.datetime_utc, responses.input_tokens, responses.output_tokens, responses.token_details, conversations.name as conversation_name, conversations.model as conversation_model, schemas.content as schema_json{rank_select} from responses left join schemas on responses.schema_id = schemas.id left join conversations on responses.conversation_id = conversations.id{join} where responses.id not in (select id from turns){extra_where} order by {order_by}{limit} """ def legacy_log_rows( db, *, count: int | None = None, model_id: str | None = None, thread_id: str | None = None, fragment_hashes=(), tool_names=(), any_tools: bool = False, schema_id: str | None = None, id_gt: str | None = None, id_gte: str | None = None, ids=(), query: str | None = None, latest: bool = False, ) -> list[dict]: """Rows for `llm logs` that exist only in the legacy tables. Applies the same filters as log_rows, translated to the legacy schema. Thread ids are conversation ids, so the two filters match the same conversations on either side of the upgrade. """ if query and "responses_fts" not in db.table_names(): # A database that never saw legacy llm has no legacy index. return [] where: list[str] = [] params: dict[str, Any] = {} if model_id: where.append("(responses.model = :model or responses.resolved_model = :model)") params["model"] = model_id if thread_id: where.append("responses.conversation_id = :thread_id") params["thread_id"] = thread_id if id_gt: where.append("responses.id > :id_gt") params["id_gt"] = id_gt if id_gte: where.append("responses.id >= :id_gte") params["id_gte"] = id_gte if schema_id: where.append("responses.schema_id = :schema_id") params["schema_id"] = schema_id if ids: keys = [f"row_id_{index}" for index in range(len(ids))] where.append( "responses.id in ({})".format(", ".join(f":{key}" for key in keys)) ) params.update(dict(zip(keys, ids))) for index, fragment_hash in enumerate(fragment_hashes): key = f"fragment_{index}" where.append(f"""( exists ( select 1 from prompt_fragments where prompt_fragments.response_id = responses.id and prompt_fragments.fragment_id in ( select fragments.id from fragments where hash = :{key} ) ) or exists ( select 1 from system_fragments where system_fragments.response_id = responses.id and system_fragments.fragment_id in ( select fragments.id from fragments where hash = :{key} ) ) )""") params[key] = fragment_hash if any_tools: where.append("""exists ( select 1 from tool_results where tool_results.response_id = responses.id )""") for index, tool_name in enumerate(tool_names): key = f"tool_{index}" where.append(f"""exists ( select 1 from tool_results join tools on tools.id = tool_results.tool_id where tool_results.response_id = responses.id and tools.name = :{key} )""") params[key] = tool_name rank_select = "" join = "" order_by = "responses.id desc" if query: rank_select = f",\n {LEGACY_SEARCH_RANK} as _search_rank" join = "\njoin responses_fts on responses_fts.rowid = responses.rowid" where.append("responses_fts match :query") params["query"] = query if not latest: order_by = LEGACY_SEARCH_RANK sql = LEGACY_LOG_ROWS_SQL.format( rank_select=rank_select, join=join, extra_where=(" and " + " and ".join(where)) if where else "", order_by=order_by, limit=f" limit {count}" if count else "", ) rows = [dict(row) for row in db.query(sql, params)] for row in rows: row["_legacy"] = True return rows def merged_log_rows( store: "LogStore", *, count: int | None = None, query: str | None = None, latest: bool = False, **filters, ): """Rows for `llm logs`: the new tables plus legacy-only responses. Both sides are newest-first and ids are ULIDs on both sides of the upgrade, so a straight sort interleaves the two histories chronologically and the top ``count`` of the union is always within the top ``count`` of each side. With ``query``, each side returns its best matches and the union is ordered most-relevant first. The two bm25 scores come from separate indexes so the interleave is approximate - close enough in practice, since both index the same kind of corpus with the same tokenizer. ``latest`` makes the query a pure filter and keeps recency order. """ rows = log_rows(store, count=count, query=query, latest=latest, **filters) rows.extend( legacy_log_rows(store.db, count=count, query=query, latest=latest, **filters) ) if query and not latest: rows.sort(key=lambda row: row["_search_rank"]) else: rows.sort(key=lambda row: row["id"], reverse=True) if count: rows = rows[:count] return rows LEGACY_ATTACHMENTS_SQL = """ select response_id, attachments.id, attachments.type, attachments.path, attachments.url, length(attachments.content) as content_length from attachments join prompt_attachments on attachments.id = prompt_attachments.attachment_id where prompt_attachments.response_id in ({placeholders}) order by prompt_attachments."order" """ LEGACY_FRAGMENTS_SQL = """ select {table}.response_id, fragments.hash, fragments.id as fragment_id, fragments.content, ( select json_group_array(fragment_aliases.alias) from fragment_aliases where fragment_aliases.fragment_id = fragments.id ) as aliases from {table} join fragments on {table}.fragment_id = fragments.id where {table}.response_id in ({placeholders}) order by {table}."order" """ LEGACY_TOOLS_SQL = """ select responses.id, coalesce( (select json_group_array(json_object( 'id', t.id, 'hash', t.hash, 'name', t.name, 'description', t.description, 'input_schema', json(t.input_schema), 'instance', null )) from tools t join tool_responses tr on t.id = tr.tool_id where tr.response_id = responses.id ), '[]' ) as tools, coalesce( (select json_group_array(json_object( 'id', tc.id, 'tool_id', tc.tool_id, 'name', tc.name, 'arguments', json(tc.arguments), 'tool_call_id', tc.tool_call_id )) from tool_calls tc where tc.response_id = responses.id ), '[]' ) as tool_calls, coalesce( (select json_group_array(json_object( 'id', tr.id, 'tool_id', tr.tool_id, 'name', tr.name, 'output', tr.output, 'tool_call_id', tr.tool_call_id, 'exception', tr.exception, 'instance', case when ti.id is not null then json_object( 'name', ti.name, 'plugin', ti.plugin, 'arguments', ti.arguments ) else null end, 'attachments', coalesce( (select json_group_array(json_object( 'id', a.id, 'type', a.type, 'path', a.path, 'url', a.url, 'content', a.content )) from tool_results_attachments tra join attachments a on tra.attachment_id = a.id where tra.tool_result_id = tr.id ), '[]' ) )) from tool_results tr left join tool_instances ti on tr.instance_id = ti.id where tr.response_id = responses.id ), '[]' ) as tool_results from responses where id in ({placeholders}) """ def legacy_log_row_extras(db, ids: list[str]) -> dict[str, dict]: """Extras for legacy rows, batch-fetched, keyed by response id. Same shape as log_row_extras, with each entry shaped the way the pre-turns `llm logs` rendered it. """ extras: dict[str, dict] = { id: { "attachments": [], "prompt_fragments": [], "system_fragments": [], "tools": [], "tool_calls": [], "tool_results": [], } for id in ids } if not ids: return extras placeholders = ",".join("?" * len(ids)) for attachment in db.query( LEGACY_ATTACHMENTS_SQL.format(placeholders=placeholders), ids ): attachment = dict(attachment) response_id = attachment.pop("response_id") extras[response_id]["attachments"].append(attachment) for table in ("prompt_fragments", "system_fragments"): for fragment in db.query( LEGACY_FRAGMENTS_SQL.format(table=table, placeholders=placeholders), ids ): fragment = dict(fragment) response_id = fragment.pop("response_id") extras[response_id][table].append(fragment) for row in db.query(LEGACY_TOOLS_SQL.format(placeholders=placeholders), ids): extras[row["id"]].update( { "tools": json.loads(row["tools"]), "tool_calls": json.loads(row["tool_calls"]), "tool_results": json.loads(row["tool_results"]), } ) return extras ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/migrations.py0000644000175100017510000005256415234420023015211 0ustar00runnerrunnerimport datetime from collections.abc import Callable MIGRATIONS: list[Callable] = [] migration = MIGRATIONS.append def migrate(db): ensure_migrations_table(db) already_applied = {r["name"] for r in db["_llm_migrations"].rows} for fn in MIGRATIONS: name = fn.__name__ if name not in already_applied: fn(db) db["_llm_migrations"].insert( { "name": name, "applied_at": str(datetime.datetime.now(datetime.timezone.utc)), } ) already_applied.add(name) def ensure_migrations_table(db): if not db["_llm_migrations"].exists(): db["_llm_migrations"].create( { "name": str, "applied_at": str, }, pk="name", ) @migration def m001_initial(db): # Ensure the original table design exists, so other migrations can run if db["log"].exists(): # It needs to have the chat_id column if "chat_id" not in db["log"].columns_dict: db["log"].add_column("chat_id") return db["log"].create( { "provider": str, "system": str, "prompt": str, "chat_id": str, "response": str, "model": str, "timestamp": str, } ) @migration def m002_id_primary_key(db): db["log"].transform(pk="id") @migration def m003_chat_id_foreign_key(db): db["log"].transform(types={"chat_id": int}) db["log"].add_foreign_key("chat_id", "log", "id") @migration def m004_column_order(db): db["log"].transform( column_order=( "id", "model", "timestamp", "prompt", "system", "response", "chat_id", ) ) @migration def m004_drop_provider(db): db["log"].transform(drop=("provider",)) @migration def m005_debug(db): db["log"].add_column("debug", str) db["log"].add_column("duration_ms", int) @migration def m006_new_logs_table(db): columns = db["log"].columns_dict for column, type in ( ("options_json", str), ("prompt_json", str), ("response_json", str), ("reply_to_id", int), ): # It's possible people running development code like myself # might have accidentally created these columns already if column not in columns: db["log"].add_column(column, type) # Use .transform() to rename options and timestamp_utc, and set new order db["log"].transform( column_order=( "id", "model", "prompt", "system", "prompt_json", "options_json", "response", "response_json", "reply_to_id", "chat_id", "duration_ms", "timestamp_utc", ), rename={ "timestamp": "timestamp_utc", "options": "options_json", }, ) @migration def m007_finish_logs_table(db): db["log"].transform( drop={"debug"}, rename={"timestamp_utc": "datetime_utc"}, drop_foreign_keys=("chat_id",), ) db.execute("alter table log rename to logs") @migration def m008_reply_to_id_foreign_key(db): db["logs"].add_foreign_key("reply_to_id", "logs", "id") @migration def m008_fix_column_order_in_logs(db): # reply_to_id ended up at the end after foreign key added db["logs"].transform( column_order=( "id", "model", "prompt", "system", "prompt_json", "options_json", "response", "response_json", "reply_to_id", "chat_id", "duration_ms", "timestamp_utc", ), ) @migration def m009_delete_logs_table_if_empty(db): # We moved to a new table design, but we don't delete the table # if someone has put data in it if not db["logs"].count: db["logs"].drop() @migration def m010_create_new_log_tables(db): db["conversations"].create( { "id": str, "name": str, "model": str, }, pk="id", ) db["responses"].create( { "id": str, "model": str, "prompt": str, "system": str, "prompt_json": str, "options_json": str, "response": str, "response_json": str, "conversation_id": str, "duration_ms": int, "datetime_utc": str, }, pk="id", foreign_keys=(("conversation_id", "conversations", "id"),), ) @migration def m011_fts_for_responses(db): db["responses"].enable_fts(["prompt", "response"], create_triggers=True) @migration def m012_attachments_tables(db): db["attachments"].create( { "id": str, "type": str, "path": str, "url": str, "content": bytes, }, pk="id", ) db["prompt_attachments"].create( { "response_id": str, "attachment_id": str, "order": int, }, foreign_keys=( ("response_id", "responses", "id"), ("attachment_id", "attachments", "id"), ), pk=("response_id", "attachment_id"), ) @migration def m013_usage(db): db["responses"].add_column("input_tokens", int) db["responses"].add_column("output_tokens", int) db["responses"].add_column("token_details", str) @migration def m014_schemas(db): db["schemas"].create( { "id": str, "content": str, }, pk="id", ) db["responses"].add_column("schema_id", str, fk="schemas", fk_col="id") # Clean up SQL create table indentation db["responses"].transform() # These changes may have dropped the FTS configuration, fix that db["responses"].enable_fts( ["prompt", "response"], create_triggers=True, replace=True ) @migration def m015_fragments_tables(db): db["fragments"].create( { "id": int, "hash": str, "content": str, "datetime_utc": str, "source": str, }, pk="id", ) db["fragments"].create_index(["hash"], unique=True) db["fragment_aliases"].create( { "alias": str, "fragment_id": int, }, foreign_keys=(("fragment_id", "fragments", "id"),), pk="alias", ) db["prompt_fragments"].create( { "response_id": str, "fragment_id": int, "order": int, }, foreign_keys=( ("response_id", "responses", "id"), ("fragment_id", "fragments", "id"), ), pk=("response_id", "fragment_id"), ) db["system_fragments"].create( { "response_id": str, "fragment_id": int, "order": int, }, foreign_keys=( ("response_id", "responses", "id"), ("fragment_id", "fragments", "id"), ), pk=("response_id", "fragment_id"), ) @migration def m016_fragments_table_pks(db): # The same fragment can be attached to a response multiple times # https://github.com/simonw/llm/issues/863#issuecomment-2781720064 db["prompt_fragments"].transform(pk=("response_id", "fragment_id", "order")) db["system_fragments"].transform(pk=("response_id", "fragment_id", "order")) @migration def m017_tools_tables(db): db["tools"].create( { "id": int, "hash": str, "name": str, "description": str, "input_schema": str, }, pk="id", ) db["tools"].create_index(["hash"], unique=True) # Many-to-many relationship between tools and responses db["tool_responses"].create( { "tool_id": int, "response_id": str, }, foreign_keys=( ("tool_id", "tools", "id"), ("response_id", "responses", "id"), ), pk=("tool_id", "response_id"), ) # tool_calls and tool_results are one-to-many against responses db["tool_calls"].create( { "id": int, "response_id": str, "tool_id": int, "name": str, "arguments": str, "tool_call_id": str, }, pk="id", foreign_keys=( ("response_id", "responses", "id"), ("tool_id", "tools", "id"), ), ) db["tool_results"].create( { "id": int, "response_id": str, "tool_id": int, "name": str, "output": str, "tool_call_id": str, }, pk="id", foreign_keys=( ("response_id", "responses", "id"), ("tool_id", "tools", "id"), ), ) @migration def m017_tools_plugin(db): db["tools"].add_column("plugin") @migration def m018_tool_instances(db): # Used to track instances of Toolbox classes that may be # used multiple times by different tools db["tool_instances"].create( { "id": int, "plugin": str, "name": str, "arguments": str, }, pk="id", ) # We record which instance was used only on the results db["tool_results"].add_column("instance_id", fk="tool_instances") @migration def m019_resolved_model(db): # For models like gemini-1.5-flash-latest where we wish to record # the resolved model name in addition to the alias db["responses"].add_column("resolved_model", str) @migration def m020_tool_results_attachments(db): db["tool_results_attachments"].create( { "tool_result_id": int, "attachment_id": str, "order": int, }, foreign_keys=( ("tool_result_id", "tool_results", "id"), ("attachment_id", "attachments", "id"), ), pk=("tool_result_id", "attachment_id"), ) @migration def m021_tool_results_exception(db): db["tool_results"].add_column("exception", str) @migration def m022_response_reasoning(db): # Concatenated visible reasoning text emitted during the response. # NULL/empty when no reasoning was emitted or when the provider # only reported an opaque token count (the redacted-marker case). db["responses"].add_column("reasoning", str) @migration def m023_message_store(db): # The content-addressed message store db["messages"].create( { "hash": str, "parent_hash": str, "role": str, "provider_metadata": str, }, pk="hash", foreign_keys=(("parent_hash", "messages", "hash"),), ) # Needed by the recursive descent through the tree and by the # child count that identifies a fork. db["messages"].create_index(["parent_hash"]) db["parts"].create( { "id": int, "message_hash": str, "position": int, # text | reasoning | tool_call | tool_result | attachment "type": str, # Tool name for tool_call and tool_result parts, else NULL. "tool_name": str, # The part's literal text, for text and reasoning parts whose # text borrows no fragments. Raw and unescaped - this column # is never parsed as anything. "text": str, # The part's remaining structure as JSON, with large content # replaced by references (fragment ids for text, attachment # ids for binary) and the type key left to the column above. # NULL when the text column carries everything. "payload": str, }, pk="id", foreign_keys=(("message_hash", "messages", "hash"),), ) # The hot read path, and it enforces one part per position. db["parts"].create_index(["message_hash", "position"], unique=True) # Attachment and fragment ids are in the payload too. These tables # exist so referential integrity, garbage collection reachability and # "everything that used X" are plain joins rather than a json_each # over every payload in the database. db["part_attachments"].create( { "part_id": int, "attachment_id": str, "order": int, }, pk=("part_id", "attachment_id", "order"), foreign_keys=( ("part_id", "parts", "id"), ("attachment_id", "attachments", "id"), ), ) db["part_fragments"].create( { "part_id": int, "fragment_id": int, "order": int, }, pk=("part_id", "fragment_id", "order"), foreign_keys=( ("part_id", "parts", "id"), ("fragment_id", "fragments", "id"), ), ) db["part_fragments"].create_index(["fragment_id"]) # The only mutable rows in the schema. A fork is a second thread # pointing at a message that already exists. db["threads"].create( { "id": str, "name": str, "tip_message_hash": str, "forked_from": str, "datetime_utc": str, }, pk="id", foreign_keys=( ("tip_message_hash", "messages", "hash"), ("forked_from", "threads", "id"), ), ) # One model call. Self-contained - it does not read anything from the # older responses table. parent_ and tip_message_hash together # delimit what this turn contributed, which nothing else records, # because message rows are shared and cannot carry provenance. db["turns"].create( { "id": str, "thread_id": str, "parent_message_hash": str, "tip_message_hash": str, "model": str, "resolved_model": str, "options_json": str, "schema_id": str, "input_tokens": int, "output_tokens": int, "token_details": str, "duration_ms": int, "datetime_utc": str, }, pk="id", foreign_keys=( ("thread_id", "threads", "id"), ("parent_message_hash", "messages", "hash"), ("tip_message_hash", "messages", "hash"), ("schema_id", "schemas", "id"), ), ) db["turns"].create_index(["thread_id"]) db["turn_tools"].create( {"turn_id": str, "tool_id": int}, pk=("turn_id", "tool_id"), foreign_keys=(("turn_id", "turns", "id"), ("tool_id", "tools", "id")), ) # Provenance: which fragments this call was given. Distinct from # part_fragments, which says what a message's text is built from. db["turn_fragments"].create( { "turn_id": str, "fragment_id": int, "order": int, "kind": str, # 'prompt' | 'system' }, pk=("turn_id", "fragment_id", "kind", "order"), foreign_keys=( ("turn_id", "turns", "id"), ("fragment_id", "fragments", "id"), ), ) db["turn_fragments"].create_index(["fragment_id"]) # Searchable text per turn: the user's typed prompt (fragment # content excluded) and the assistant's text output, kept fresh by # LogStore.log. An explicit id primary key because external-content # FTS is keyed by rowid, and implicit rowids are not stable across # VACUUM. db["turn_search"].create( { "id": int, "turn_id": str, "prompt": str, "response": str, }, pk="id", foreign_keys=(("turn_id", "turns", "id"),), ) db["turn_search"].create_index(["turn_id"], unique=True) db["turn_search"].enable_fts(["prompt", "response"], create_triggers=True) # Which configured toolbox instance served a tool call: the toolbox # name, its plugin and its constructor arguments. Local execution # provenance, so it lives outside the hashed message tree, joined # to the chain by tool_call_id - and keyed by (turn_id, # tool_call_id), because provider-supplied call ids are not # guaranteed unique across turns. Deliberately the seed of a fuller # execution-events table: duration or exception details would be # additive columns here. db["tool_instantiations"].create( { "turn_id": str, "tool_call_id": str, "name": str, "plugin": str, "arguments": str, }, pk=("turn_id", "tool_call_id"), foreign_keys=(("turn_id", "turns", "id"),), ) @migration def m024_tool_instance_references(db): # Tool instance configurations - e.g. Datasette("https://...") - # are stored once in the shared tool_instances table and referenced # by id, instead of being copied onto every row that mentions them: # tool_instantiations gains instance_id in place of its # name/plugin/arguments copies, and turn_tools gains instance_id so # the tools list can show which configured instance provided each # tool. from .utils import ensure_tool_instance db["turn_tools"].add_column("instance_id", int, fk="tool_instances", fk_col="id") db["tool_instantiations"].add_column( "instance_id", int, fk="tool_instances", fk_col="id" ) for row in list(db["tool_instantiations"].rows): db["tool_instantiations"].update( (row["turn_id"], row["tool_call_id"]), { "instance_id": ensure_tool_instance( db, row["name"], row["plugin"], row["arguments"] ) }, ) db["tool_instantiations"].transform(drop={"name", "plugin", "arguments"}) @migration def m025_turn_tools_instance_backfill(db): # turn_tools rows written before instance_id existed have NULL # there, so the tools list shows nothing for them. The instance # that served calls in the same thread, matched by toolbox name # prefix, is the right value for these development-era rows. with db.atomic(): db.execute(""" update turn_tools set instance_id = ( select ti.instance_id from tool_instantiations ti join turns turn_a on turn_a.id = ti.turn_id join turns turn_b on turn_b.id = turn_tools.turn_id and turn_b.thread_id = turn_a.thread_id join tool_instances instance on instance.id = ti.instance_id join tools on tools.id = turn_tools.tool_id where tools.name = instance.name or tools.name like instance.name || '\\_%' escape '\\' limit 1 ) where instance_id is null """) MESSAGE_TREE_SQL = """ with recursive msg as ( select m.hash, m.parent_hash, m.role, m.rowid as rid, replace(coalesce( nullif(p.text, ''), (select f.content from part_fragments pf join fragments f on f.id = pf.fragment_id where pf.part_id = p.id order by pf."order" limit 1), '[' || coalesce(p.type, 'empty') || ']' ), char(10), ' ') as text, (select group_concat(p2.tool_name, ', ') from parts p2 where p2.message_hash = m.hash and p2.type = 'tool_result' and p2.tool_name is not null) as tools from messages m left join parts p on p.message_hash = m.hash and p.position = 0 ), tree as ( select hash, text, tools, 0 as depth, printf('%012d', rid) as path, hash as root_hash from msg where parent_hash is null union all select msg.hash, msg.text, msg.tools, t.depth + 1, t.path || '/' || printf('%012d', msg.rid), t.root_hash from msg join tree t on msg.parent_hash = t.hash ), turn_chain as ( select t.id as turn_id, t.datetime_utc, m.hash, m.parent_hash from turns t join messages m on m.hash = t.tip_message_hash union all select tc.turn_id, tc.datetime_utc, m.hash, m.parent_hash from turn_chain tc join messages m on m.hash = tc.parent_hash ) select t.root_hash, strftime('%Y-%m-%d %H:%M:%S', (select min(tc.datetime_utc) from turn_chain tc where tc.hash = t.hash) ) as datetime, replace(hex(zeroblob(t.depth)), '00', ' ') || substr(t.text, 1, 60) as message, coalesce(t.tools, '') as tools, t.hash as message_hash, t.path from tree t order by t.path """.strip() @migration def m026_message_tree_view(db): # A readable rendering of the message store: every conversation # tree as indented text, one row per message, depth-first with # siblings in insertion order. root_hash identifies a tree - filter # on it to isolate one conversation and its forks. datetime is the # earliest turn that recorded the message, since shared message # rows carry no timestamp of their own. Kept ordered by including # path in the output - selecting from the view preserves tree order # only while sorted by path. db.create_view("message_tree", MESSAGE_TREE_SQL) @migration def m027_turns_response_json(db): # The raw provider payload for the call, stored condensed: strings # that already live in the turn's message parts (response text, # reasoning blobs, long tool arguments) are replaced with references # via condense-json, so the column costs roughly the provider # envelope - ids, usage, fingerprints - not a second copy of the # response. NULL for turns logged before this column existed and # for models that expose no raw payload. db["turns"].add_column("response_json", str) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/models.py0000644000175100017510000040336515234420023014317 0ustar00runnerrunnerimport asyncio import base64 import dataclasses import datetime import functools import hashlib import re import time from collections.abc import ( AsyncGenerator, AsyncIterator, Awaitable, Callable, Iterable, Iterator, ) from dataclasses import dataclass, field from itertools import islice from pathlib import Path from types import MethodType from typing import ( TYPE_CHECKING, Any, ClassVar, Optional, Union, cast, get_type_hints, ) import httpx from .errors import NeedsKeyException from .serialization import ResponseDict if TYPE_CHECKING: from .parts import StreamEvent import inspect import json from abc import ABC, abstractmethod from pydantic import BaseModel, ConfigDict, create_model from .utils import ( Fragment, mimetype_from_path, mimetype_from_string, monotonic_ulid, token_usage_string, ) CONVERSATION_NAME_LENGTH = 32 @dataclass class Usage: "Token usage information from a model response." input: int | None = None output: int | None = None details: dict[str, Any] | None = None @dataclass class Attachment: "An attachment (image, audio, etc) to include with a prompt." type: str | None = None path: str | None = None url: str | None = None content: bytes | None = None _id: str | None = None def id(self): # Hash of the binary content, or of '{"url": "https://..."}' for URL attachments if self._id is None: if self.content: self._id = hashlib.sha256(self.content).hexdigest() elif self.path: self._id = hashlib.sha256(Path(self.path).read_bytes()).hexdigest() else: self._id = hashlib.sha256( json.dumps({"url": self.url}).encode("utf-8") ).hexdigest() return self._id def resolve_type(self): "Return the content type, guessing from content if not specified." if self.type: return self.type # Derive it from path or url or content if self.path: return mimetype_from_path(self.path) if self.url: with httpx.Client(follow_redirects=True, max_redirects=3) as client: response = client.head(self.url) response.raise_for_status() return response.headers.get("content-type") if self.content: return mimetype_from_string(self.content) raise ValueError("Attachment has no type and no content to derive it from") def content_bytes(self): "Return the binary content, reading from path or URL if needed." content = self.content if not content: if self.path: content = Path(self.path).read_bytes() elif self.url: with httpx.Client(follow_redirects=True, max_redirects=3) as client: response = client.get(self.url) response.raise_for_status() content = response.content return content def base64_content(self): "Return the content as a base64-encoded string." return base64.b64encode(self.content_bytes()).decode("utf-8") def __repr__(self): info = [f"" @classmethod def from_row(cls, row): return cls( _id=row["id"], type=row["type"], path=row["path"], url=row["url"], content=row["content"], ) @dataclass class Tool: "A tool that can be called by a model." name: str description: str | None = None input_schema: dict = field(default_factory=dict) implementation: Callable | None = None plugin: str | None = None # plugin tool came from, e.g. 'llm_tools_sqlite' def __post_init__(self): # Convert Pydantic model to JSON schema if needed self.input_schema = _ensure_dict_schema(self.input_schema) def hash(self): """Hash for tool based on its name, description and input schema (preserving key order)""" to_hash = { "name": self.name, "description": self.description, "input_schema": self.input_schema, } if self.plugin: to_hash["plugin"] = self.plugin return hashlib.sha256(json.dumps(to_hash).encode("utf-8")).hexdigest() @classmethod def function(cls, function, name=None, description=None): """ Turn a Python function into a Tool object by: - Extracting the function name - Using the function docstring for the Tool description - Building a Pydantic model for inputs by inspecting the function signature - Building a Pydantic model for the return value by using the function's return annotation """ if not name and function.__name__ == "": raise ValueError( "Cannot create a Tool from a lambda function without providing name=" ) return cls( name=name or function.__name__, description=description or function.__doc__ or None, input_schema=_get_arguments_input_schema(function, name), implementation=function, ) class ServerSideTool: """A tool executed inside the provider's infrastructure. Instances are passed in ``tools=[...]`` alongside function tools. The framework transports and validates them but never executes them. Provider plugins subclass this class for their own tools; the base class can be instantiated directly with a raw provider tool specification. """ name: ClassVar[str] = "server_side_tool" plugin: ClassVar[str | None] = None implementation: ClassVar[None] = None input_schema: ClassVar[dict] = {} def __init__(self, spec: dict | None = None): self.spec = spec self._config = {"spec": spec} def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) original_init = cls.__init__ @functools.wraps(original_init) def wrapped_init(self, *args, **kwargs): signature = inspect.signature(original_init) bound = signature.bind(self, *args, **kwargs) bound.apply_defaults() original_init(self, *args, **kwargs) self._config = { name: value for name, value in bound.arguments.items() if name != "self" and signature.parameters[name].kind not in ( inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD, ) } cls.__init__ = wrapped_init @property def description(self) -> str | None: return inspect.getdoc(self.__class__) def tool_spec(self, model) -> dict: """Return this tool's provider-specific request specification.""" if self.spec is None: raise TypeError( f"{self.__class__.__name__} does not define a raw provider tool spec" ) return self.spec def prepare_request(self, model, kwargs: dict) -> None: """Add any other values this tool needs to provider request kwargs.""" def hash(self): """Hash the definition separately from configured instances.""" to_hash = { "name": self.name, "description": self.description, "input_schema": self.input_schema, "server_side": True, } if self.plugin: to_hash["plugin"] = self.plugin return hashlib.sha256(json.dumps(to_hash).encode("utf-8")).hexdigest() def _get_arguments_input_schema(function, name): signature = inspect.signature(function) type_hints = get_type_hints(function) fields = {} for param_name, param in signature.parameters.items(): if param_name in ("self", "llm_tool_call"): # llm_tool_call is reserved: populated with the ToolCall object # at execution time, never exposed to the model. continue # Determine the type annotation (default to string if missing) annotated_type = type_hints.get(param_name, str) # Handle default value if present; if there's no default, use '...' if param.default is inspect.Parameter.empty: fields[param_name] = (annotated_type, ...) else: fields[param_name] = (annotated_type, param.default) return create_model(f"{name}InputSchema", **fields) def _accepts_llm_tool_call(implementation) -> bool: try: signature = inspect.signature(implementation) except (TypeError, ValueError): return False return "llm_tool_call" in signature.parameters def _implementation_arguments(tool: "Tool", tool_call: "ToolCall") -> dict: """Arguments to invoke a tool implementation with. Implementations with an explicit ``llm_tool_call`` parameter receive the ToolCall object itself - a ``**kwargs`` catch-all does not count. """ arguments = dict(tool_call.arguments) if _accepts_llm_tool_call(tool.implementation): arguments["llm_tool_call"] = tool_call return arguments class Toolbox: name: str | None = None instance_id: int | None = None _blocked = ( "tools", "add_tool", "method_tools", "__init_subclass__", "prepare", "prepare_async", ) _extra_tools: ClassVar[list[Tool]] = [] _config: ClassVar[dict[str, Any]] = {} _prepared: bool = False _async_prepared: bool = False def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) original_init = cls.__init__ @functools.wraps(original_init) def wrapped_init(self, *args, **kwargs): # Track args/kwargs passed to constructor in self._config # so we can serialize them to a database entry later on sig = inspect.signature(original_init) bound = sig.bind(self, *args, **kwargs) bound.apply_defaults() self._config = { name: value for name, value in bound.arguments.items() if name != "self" and sig.parameters[name].kind not in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD) } self._extra_tools = [] original_init(self, *args, **kwargs) cls.__init__ = wrapped_init @classmethod def method_tools(cls) -> list[Tool]: tools = [] for method_name in dir(cls): if method_name.startswith("_") or method_name in cls._blocked: continue method = getattr(cls, method_name) if callable(method): tool = Tool.function( method, name=f"{cls.__name__}_{method_name}", ) tools.append(tool) return tools def tools(self) -> Iterable[Tool]: "Returns an llm.Tool() for each class method, plus any extras registered with add_tool()" # method_tools() returns unbound methods, we need bound methods here: for name in dir(self): if name.startswith("_") or name in self._blocked: continue attr = getattr(self, name) if callable(attr): tool = Tool.function(attr, name=f"{self.__class__.__name__}_{name}") tool.plugin = getattr(self, "plugin", None) yield tool yield from self._extra_tools def add_tool( self, tool_or_function: Tool | Callable[..., Any], pass_self: bool = False ): "Add a tool to this toolbox" def _upgrade(fn): if pass_self: return MethodType(fn, self) return fn if isinstance(tool_or_function, Tool): self._extra_tools.append(tool_or_function) elif callable(tool_or_function): self._extra_tools.append(Tool.function(_upgrade(tool_or_function))) else: raise TypeError("Tool must be an instance of Tool or a callable function") def prepare(self): """ Over-ride this to perform setup (and .add_tool() calls) before the toolbox is used. Implement a similar prepare_async() method for async setup. """ async def prepare_async(self): """ Over-ride this to perform async setup (and .add_tool() calls) before the toolbox is used. """ @dataclass class ToolCall: "A request by the model to call a tool." name: str arguments: dict tool_call_id: str | None = None def _ensure_tool_call_id(tool_call: ToolCall) -> ToolCall: # Generate a tool call ID if one has not yet been specified if tool_call.tool_call_id is not None: return tool_call return dataclasses.replace( tool_call, tool_call_id=f"tc_{str(monotonic_ulid()).lower()}", ) @dataclass class ToolResult: "The result of executing a tool call." name: str output: str attachments: list[Attachment] = field(default_factory=list) tool_call_id: str | None = None instance: Toolbox | None = None exception: Exception | None = None @dataclass class ToolOutput: "Tool functions can return output with extra attachments" output: str | dict | list | bool | int | float | None = None attachments: list[Attachment] = field(default_factory=list) ToolDef = Tool | Toolbox | ServerSideTool | Callable[..., Any] BeforeCallSync = Callable[[Tool | None, ToolCall], None] AfterCallSync = Callable[[Tool, ToolCall, ToolResult], None] BeforeCallAsync = Callable[[Tool | None, ToolCall], None | Awaitable[None]] AfterCallAsync = Callable[[Tool, ToolCall, ToolResult], None | Awaitable[None]] class CancelToolCall(Exception): pass class PauseChain(Exception): """Raise inside a tool implementation to pause the chain. Unlike other exceptions - which are converted into error ToolResults and sent back to the model - PauseChain propagates out of ``execute_tool_calls()`` and ``chain()``. Before it is re-raised the framework populates two attributes: - ``tool_call``: the ToolCall whose implementation paused - ``tool_results``: ToolResults of sibling calls in the same batch that completed Concurrent (async) sibling tool calls always run to completion before the exception propagates; sequential (sync) execution stops at the paused call, leaving later calls unexecuted so they can safely run when the chain is resumed. Resume by re-running the chain with a ``messages=`` history that ends in the unresolved tool calls. """ def __init__(self, *args): super().__init__(*args) self.tool_call: ToolCall | None = None self.tool_results: list[ToolResult] = [] @dataclass class Prompt: "The prompt being sent to the model." _prompt: str | None model: "Model" fragments: list[str | Fragment] | None attachments: list[Attachment] | None _system: str | None system_fragments: list[str | Fragment] | None prompt_json: str | None schema: dict | type[BaseModel] | None tools: list[Tool | ServerSideTool] tool_results: list[ToolResult] options: "Options" hide_reasoning: bool def __init__( self, prompt, model, *, fragments=None, attachments=None, system=None, system_fragments=None, prompt_json=None, options=None, schema=None, tools=None, tool_results=None, messages=None, hide_reasoning=False, ): self._prompt = prompt self.model = model self.attachments = list(attachments or []) self.fragments = fragments or [] self._system = system self.system_fragments = system_fragments or [] self.prompt_json = prompt_json if schema and not isinstance(schema, dict) and issubclass(schema, BaseModel): schema = schema.model_json_schema() self.schema = schema self.tools = _wrap_tools(tools or []) self.tool_results = tool_results or [] self.options = options or {} self.hide_reasoning = hide_reasoning # Explicit messages= list, if the caller supplied one. Copied so # later mutation by the caller doesn't alter the Prompt. self._explicit_messages = list(messages) if messages is not None else None @property def prompt(self): "The text of the prompt, with any fragments concatenated." return "\n".join(self.fragments + ([self._prompt] if self._prompt else [])) @property def system(self): "The system prompt, with any system fragments concatenated." return _combine_system(self._system, self.system_fragments) @property def messages(self): """Canonical list of Message objects for this prompt. **Invariant:** this property returns exactly what the model was (or will be) sent for this turn — the full chain including any prior conversation history. - If ``messages=`` was passed explicitly, it is authoritative: returned verbatim. Other kwargs (``prompt=``, ``system=``, ``attachments=``, ``tool_results=``) are ignored for the messages list (they remain available via ``prompt.prompt``, ``prompt.system``, etc., for adapters that still read them). - Otherwise the list is synthesized from the legacy kwargs (system, tool_results, prompt, attachments), producing just the current turn — prior history is not folded in, because no conversation context is reachable here. Conversation.prompt / AsyncConversation.prompt / reply() all pre-compute the full chain and pass it as ``messages=``, so ``response.prompt.messages`` after those paths is the full chain. """ from .parts import ( AttachmentPart, Message, TextPart, ToolResultPart, ) if self._explicit_messages is not None: return list(self._explicit_messages) result: list[Message] = [] if self.system: result.append(Message(role="system", parts=[TextPart(text=self.system)])) if self.tool_results: result.append( Message( role="tool", parts=[ ToolResultPart( name=tr.name, output=tr.output, tool_call_id=tr.tool_call_id, exception=_format_tool_exception(tr.exception), attachments=list(tr.attachments or []), ) for tr in self.tool_results ], ) ) user_parts: list[Any] = [] if self.prompt: user_parts.append(TextPart(text=self.prompt)) for att in self.attachments: user_parts.append(AttachmentPart(attachment=att)) if user_parts: result.append(Message(role="user", parts=user_parts)) return result def _wrap_tools(tools: list[ToolDef]) -> list[Tool | ServerSideTool]: wrapped_tools = [] for tool in tools: if isinstance(tool, (Tool, ServerSideTool)): wrapped_tools.append(tool) elif isinstance(tool, Toolbox): wrapped_tools.extend(tool.tools()) elif callable(tool): wrapped_tools.append(Tool.function(tool)) else: raise TypeError(f"Invalid tool: {tool}") return wrapped_tools def _partition_tools( model: "_BaseModel", tools: Iterable[Tool | ServerSideTool] ) -> tuple[list[Tool], list[ServerSideTool]]: """Partition tools and reject server-side tools the model did not claim.""" function_tools = [] server_side_tools = [] declared = tuple(model.supported_server_side_tools) for tool in tools: if isinstance(tool, ServerSideTool): # Declaring ServerSideTool itself claims only direct raw-spec # instances. Without this exact-type exception it would also # accidentally claim every provider-specific subclass. claimed = any( ( type(tool) is candidate if candidate is ServerSideTool else isinstance(tool, candidate) ) for candidate in declared ) if not claimed: raise ValueError( f"Model '{model.model_id}' does not support server-side tool " f"'{tool.name}'. Run: llm tools -m {model.model_id}" ) server_side_tools.append(tool) else: function_tools.append(tool) return function_tools, server_side_tools def _append_turn_input( chain: list[Any], prompt: str | None, fragments=None, attachments=None, tool_results=None, ) -> list[Any]: """Append a turn's new input to a message chain. A tool-role message for any tool results, then a user-role message built from fragments + prompt text + attachments. This is what makes ``messages=`` mean "authoritative history" without the other prompt arguments being silently dropped from the chain. Mutates and returns ``chain``. """ from .parts import ( AttachmentPart, Message, TextPart, ToolResultPart, ) if tool_results: chain.append( Message( role="tool", parts=[ ToolResultPart( name=tr.name, output=tr.output, tool_call_id=tr.tool_call_id, exception=_format_tool_exception(tr.exception), attachments=list(tr.attachments or []), ) for tr in tool_results ], ) ) user_parts: list[Any] = [] # Fragments are concatenated into the prompt text before it is # sent, so they have to be in the chain too - prompt.messages is # meant to be exactly what the model sees. Matches Prompt.prompt. prompt_text = "\n".join( [str(fragment) for fragment in fragments or []] + ([prompt] if prompt else []) ) if prompt_text: user_parts.append(TextPart(text=prompt_text)) for att in attachments or []: user_parts.append(AttachmentPart(attachment=att)) if user_parts: chain.append(Message(role="user", parts=user_parts)) return chain def _combine_system(system, system_fragments): "Concatenate the system prompt and any system fragments into one string." bits = [ bit.strip() for bit in ((system_fragments or []) + [system or ""]) if bit.strip() ] return "\n\n".join(bits) def _merge_options(options: dict | None, kwargs: dict) -> dict: if not options: return kwargs overlap = set(options) & set(kwargs) if overlap: raise TypeError( "Got values for these options both in options= and as keyword " f"arguments: {sorted(overlap)}" ) return {**options, **kwargs} @dataclass class _BaseConversation: model: "_BaseModel" id: str = field(default_factory=lambda: str(monotonic_ulid()).lower()) name: str | None = None responses: list["_BaseResponse"] = field(default_factory=list) tools: list[ToolDef] | None = None chain_limit: int | None = None # History read back from storage, used as the chain for the next turn # when this conversation has not yet produced a response in this # process. Unlike a chain rebuilt from logged responses this is the # exact message list, so reasoning signatures and provider metadata # survive being reloaded. loaded_messages: list[Any] | None = None # Plugin and server-side tool names and configured specs (e.g. # 'Datasette({"url": ...})' or 'CodeInterpreter({"memory_limit": # "4g"})') recorded against this conversation's first turn in storage. # Read when the conversation was loaded from the message store, where # there are no rebuilt responses to copy prompt.tools from. loaded_tools: list[str] | None = None @classmethod @abstractmethod def from_row(cls, row: Any) -> "_BaseConversation": raise NotImplementedError def _record_response(self, response: "_BaseResponse") -> None: "Record a completed response as part of this conversation." self.responses.append(response) # History now comes from the live responses, so anything read # back from storage is superseded. self.loaded_messages = None def _build_full_chain( self, prompt: str | None, attachments, tool_results, explicit_messages, system=None, system_fragments=None, fragments=None, ) -> list[Any]: """Build the full message chain for the next turn. Uses the last response's stored prompt chain to recover prior history, then appends the new turn's content (explicit messages first, or synthesized from prompt/attachments/tool_results). Returns the list that should be passed as ``messages=`` to the Prompt constructor so that ``response.prompt.messages`` equals exactly what the model sees. If ``explicit_messages`` is provided, the caller has opted out of history reconstruction: the list is the authoritative history, and the new turn's input is appended to it. """ from .parts import Message, TextPart if explicit_messages is not None: return _append_turn_input( list(explicit_messages), prompt, fragments, attachments, tool_results, ) chain: list[Any] = [] if self.loaded_messages: # Storage holds the exact chain, so prefer it over anything # rebuilt from logged responses. Cleared as soon as this # conversation produces a response of its own. chain.extend(self.loaded_messages) elif self.responses: last = self.responses[-1] # last.prompt.messages already contains the full input chain # under the invariant, so use the last response only and then # append that response's structured output. chain.extend(last.prompt.messages) chain.extend(last._messages_now()) else: # Start with the system prompt as the first message so adapters # that build from prompt.messages see it. On later turns it # is already carried forward in last.prompt.messages. system_text = _combine_system(system, system_fragments) if system_text: chain.append(Message(role="system", parts=[TextPart(text=system_text)])) return _append_turn_input(chain, prompt, fragments, attachments, tool_results) @dataclass class Conversation(_BaseConversation): before_call: BeforeCallSync | None = None after_call: AfterCallSync | None = None def prompt( self, prompt: str | None = None, *, fragments: list[str | Fragment] | None = None, attachments: list[Attachment] | None = None, system: str | None = None, schema: dict | type[BaseModel] | None = None, tools: list[ToolDef] | None = None, tool_results: list[ToolResult] | None = None, system_fragments: list[str | Fragment] | None = None, messages: list[Any] | None = None, stream: bool = True, key: str | None = None, options: dict | None = None, hide_reasoning: bool = False, **kwargs, ) -> "Response": merged = _merge_options(options, kwargs) # Build the authoritative chain so response.prompt.messages # equals exactly what the model sees for this turn. chain = self._build_full_chain( prompt=prompt, attachments=attachments, tool_results=tool_results, explicit_messages=messages, system=system, system_fragments=system_fragments, fragments=fragments, ) return Response( Prompt( prompt, model=self.model, fragments=fragments, attachments=attachments, system=system, schema=schema, tools=tools or self.tools, tool_results=tool_results, system_fragments=system_fragments, messages=chain, options=self.model.Options(**merged), hide_reasoning=hide_reasoning, ), self.model, stream, conversation=self, key=key, before_call=self.before_call, after_call=self.after_call, ) def chain( self, prompt: str | None = None, *, fragments: list[str] | None = None, attachments: list[Attachment] | None = None, system: str | None = None, system_fragments: list[str] | None = None, messages: list[Any] | None = None, stream: bool = True, schema: dict | type[BaseModel] | None = None, tools: list[ToolDef] | None = None, tool_results: list[ToolResult] | None = None, chain_limit: int | None = None, before_call: BeforeCallSync | None = None, after_call: AfterCallSync | None = None, key: str | None = None, options: dict | None = None, hide_reasoning: bool = False, ) -> "ChainResponse": self.model._validate_attachments(attachments) # Parity with Conversation.prompt: pre-bake the full chain so # response.prompt.messages is authoritative for the first turn # of the chain loop. Subsequent tool-result turns extend the # chain via _chain_for_tool_results. chain_messages = self._build_full_chain( prompt=prompt, attachments=attachments, tool_results=tool_results, explicit_messages=messages, system=system, system_fragments=system_fragments, fragments=fragments, ) return ChainResponse( Prompt( prompt, fragments=fragments, attachments=attachments, system=system, schema=schema, tools=tools or self.tools, tool_results=tool_results, system_fragments=system_fragments, messages=chain_messages, model=self.model, options=self.model.Options(**(options or {})), hide_reasoning=hide_reasoning, ), model=self.model, stream=stream, conversation=self, key=key, before_call=before_call or self.before_call, after_call=after_call or self.after_call, chain_limit=chain_limit if chain_limit is not None else self.chain_limit, ) @classmethod def from_row(cls, row): from llm import get_model return cls( model=get_model(row["model"]), id=row["id"], name=row["name"], ) def __repr__(self): count = len(self.responses) s = "s" if count == 1 else "" return f"<{self.__class__.__name__}: {self.id} - {count} response{s}" @dataclass class AsyncConversation(_BaseConversation): before_call: BeforeCallAsync | None = None after_call: AfterCallAsync | None = None def chain( self, prompt: str | None = None, *, fragments: list[str] | None = None, attachments: list[Attachment] | None = None, system: str | None = None, system_fragments: list[str] | None = None, messages: list[Any] | None = None, stream: bool = True, schema: dict | type[BaseModel] | None = None, tools: list[ToolDef] | None = None, tool_results: list[ToolResult] | None = None, chain_limit: int | None = None, before_call: BeforeCallAsync | None = None, after_call: AfterCallAsync | None = None, key: str | None = None, options: dict | None = None, hide_reasoning: bool = False, ) -> "AsyncChainResponse": self.model._validate_attachments(attachments) chain_messages = self._build_full_chain( prompt=prompt, attachments=attachments, tool_results=tool_results, explicit_messages=messages, system=system, system_fragments=system_fragments, fragments=fragments, ) return AsyncChainResponse( Prompt( prompt, fragments=fragments, attachments=attachments, system=system, schema=schema, tools=tools or self.tools, tool_results=tool_results, system_fragments=system_fragments, messages=chain_messages, model=self.model, options=self.model.Options(**(options or {})), hide_reasoning=hide_reasoning, ), model=self.model, stream=stream, conversation=self, key=key, before_call=before_call or self.before_call, after_call=after_call or self.after_call, chain_limit=chain_limit if chain_limit is not None else self.chain_limit, ) def prompt( self, prompt: str | None = None, *, fragments: list[str] | None = None, attachments: list[Attachment] | None = None, system: str | None = None, schema: dict | type[BaseModel] | None = None, tools: list[ToolDef] | None = None, tool_results: list[ToolResult] | None = None, system_fragments: list[str] | None = None, messages: list[Any] | None = None, stream: bool = True, key: str | None = None, options: dict | None = None, hide_reasoning: bool = False, **kwargs, ) -> "AsyncResponse": merged = _merge_options(options, kwargs) chain = self._build_full_chain( prompt=prompt, attachments=attachments, tool_results=tool_results, explicit_messages=messages, system=system, system_fragments=system_fragments, fragments=fragments, ) return AsyncResponse( Prompt( prompt, model=self.model, fragments=fragments, attachments=attachments, system=system, schema=schema, tools=tools, tool_results=tool_results, system_fragments=system_fragments, messages=chain, options=self.model.Options(**merged), hide_reasoning=hide_reasoning, ), self.model, stream, conversation=self, key=key, before_call=self.before_call, after_call=self.after_call, ) def to_sync_conversation(self): return Conversation( model=self.model, id=self.id, name=self.name, responses=[], # Because we only use this in logging tools=self.tools, chain_limit=self.chain_limit, ) @classmethod def from_row(cls, row): from llm import get_async_model return cls( model=get_async_model(row["model"]), id=row["id"], name=row["name"], ) def __repr__(self): count = len(self.responses) s = "s" if count == 1 else "" return f"<{self.__class__.__name__}: {self.id} - {count} response{s}" FRAGMENT_SQL = """ select 'prompt' as fragment_type, fragments.content, pf."order" as ord from prompt_fragments pf join fragments on pf.fragment_id = fragments.id where pf.response_id = :response_id union all select 'system' as fragment_type, fragments.content, sf."order" as ord from system_fragments sf join fragments on sf.fragment_id = fragments.id where sf.response_id = :response_id order by fragment_type desc, ord asc; """ class _BaseResponse: """Base response class shared between sync and async responses""" id: str prompt: "Prompt" stream: bool resolved_model: str | None = None conversation: Optional["_BaseConversation"] = None _key: str | None = None def __init__( self, prompt: Prompt, model: "_BaseModel", stream: bool, conversation: _BaseConversation | None = None, key: str | None = None, before_call: BeforeCallSync | BeforeCallAsync | None = None, after_call: AfterCallSync | AfterCallAsync | None = None, ): self.id = str(monotonic_ulid()).lower() self.prompt = prompt self._prompt_json = None self.model = model self.stream = stream self._key = key self.before_call = before_call self.after_call = after_call self._chunks: list[str] = [] # Every StreamEvent ever yielded by execute(), in order. Plain # str yields are wrapped as text events (with part_index resolved # by _resolve_part_index) so this buffer is the single source of # truth for replay and for assembling response.messages. self._stream_events: list[Any] = [] # Auto-allocator state for resolving StreamEvent.part_index=None. # Plugins yield events with part_index=None (the default) and # the framework assigns concrete integers based on context: # consecutive same-family text/reasoning events concatenate, # tool calls group by tool_call_id, and tool_result is always # its own part. _auto_index_max tracks the highest index seen # (explicit or allocated); _auto_last_index / _auto_last_family # remember the previously-resolved event so same-family runs # share an index; _auto_tool_id_to_index maps known tool ids to # their assigned index for parallel-tool-call grouping. self._auto_index_max: int = -1 self._auto_last_index: int | None = None self._auto_last_family: str | None = None self._auto_tool_id_to_index: dict[str, int] = {} self._auto_last_message_index: int = 0 self._done = False self._tool_calls: list[ToolCall] = [] self.response_json: dict[str, Any] | None = None self.conversation = conversation self.attachments: list[Attachment] = [] self._start: float | None = None self._end: float | None = None self._start_utcnow: datetime.datetime | None = None self.input_tokens: int | None = None self.output_tokens: int | None = None self.token_details: dict | None = None self.done_callbacks: list[Callable] = [] if self.prompt.schema and not self.model.supports_schema: raise ValueError(f"{self.model} does not support schemas") function_tools, _ = _partition_tools(self.model, self.prompt.tools) if function_tools and not self.model.supports_tools: raise ValueError(f"{self.model} does not support tools") def _messages_now(self) -> list[Any]: """Assemble messages assuming the response is already drained. Public ``messages()`` forces / awaits first, then delegates here. Internal sync paths (``_response_to_dict``, ``_chain_for_tool_results``) call this directly so they don't have to await on async responses. """ from .parts import Message loaded = getattr(self, "_loaded_messages", None) if loaded is not None: return list(loaded) return [ Message(role="assistant", parts=parts) for parts in self._build_message_parts() ] @staticmethod def _event_family(event_type: str) -> str: if event_type in ("tool_call_name", "tool_call_args"): return "tool_call" return event_type def _resolve_part_index(self, event): """Mutate event.part_index in place when the plugin left it None. Resolution rules: consecutive same-family text/reasoning events share an index; tool-call events are grouped by tool_call_id; tool_result always allocates a fresh index. Explicit indices pass through but update the allocator's bookkeeping so future None resolutions avoid collisions. """ fam = self._event_family(event.type) # A message_index change always starts a fresh part - text on # either side of a message boundary must not concatenate. if event.message_index != self._auto_last_message_index: self._auto_last_family = None self._auto_last_index = None self._auto_last_message_index = event.message_index if event.part_index is not None: self._auto_index_max = max(self._auto_index_max, event.part_index) if ( event.type in ("tool_call_name", "tool_call_args") and event.tool_call_id ): self._auto_tool_id_to_index[event.tool_call_id] = event.part_index self._auto_last_index = event.part_index self._auto_last_family = fam return if event.type in ("tool_call_name", "tool_call_args"): if event.tool_call_id: existing = self._auto_tool_id_to_index.get(event.tool_call_id) if existing is not None: event.part_index = existing self._auto_last_index = existing self._auto_last_family = "tool_call" return self._auto_index_max += 1 new_idx = self._auto_index_max self._auto_tool_id_to_index[event.tool_call_id] = new_idx event.part_index = new_idx self._auto_last_index = new_idx self._auto_last_family = "tool_call" return # No tool_call_id — providers like Gemini omit the id on # parallel tool calls. tool_call_args events glue onto the # most recent tool-call index; a fresh tool_call_name # always starts a new part (otherwise N parallel tool calls # collapse into one with concatenated names and args). if ( event.type == "tool_call_args" and self._auto_last_family == "tool_call" and self._auto_last_index is not None ): event.part_index = self._auto_last_index return self._auto_index_max += 1 new_idx = self._auto_index_max event.part_index = new_idx self._auto_last_index = new_idx self._auto_last_family = "tool_call" return if event.type == "tool_result": self._auto_index_max += 1 new_idx = self._auto_index_max event.part_index = new_idx self._auto_last_index = new_idx self._auto_last_family = "tool_result" return # text / reasoning: same family as previous → reuse, else new. if self._auto_last_family == fam and self._auto_last_index is not None: event.part_index = self._auto_last_index return self._auto_index_max += 1 new_idx = self._auto_index_max event.part_index = new_idx self._auto_last_index = new_idx self._auto_last_family = fam def _process_chunk(self, chunk): """Normalize a chunk from execute() into a StreamEvent and return the text str (or None) that __iter__ should yield. Plain str yields from legacy plugins are wrapped as text events with an auto-allocated part_index. Side effects: populates self._stream_events and self._chunks. """ from .parts import StreamEvent if isinstance(chunk, StreamEvent): self._resolve_part_index(chunk) self._stream_events.append(chunk) if chunk.type == "text": self._chunks.append(chunk.chunk) return chunk.chunk return None # Legacy plain-str plugin. event = StreamEvent(type="text", chunk=chunk) self._resolve_part_index(event) self._stream_events.append(event) self._chunks.append(chunk) return chunk def _build_parts(self) -> list[Any]: """All Parts from the accumulated stream events, flattened across message boundaries. See ``_build_message_parts``.""" return [part for parts in self._build_message_parts() for part in parts] def _build_message_parts(self) -> list[list[Any]]: """Assemble Part objects from the accumulated stream events, grouped into one parts-list per assistant message. Most providers emit a single assistant message, so the result is usually a one-element list. Events carrying an explicit ``message_index`` (OpenAI Responses server-side tool execution interleaves multiple ``message`` output items in one response) split into one parts-list per distinct index, in first-seen order. Events sharing a part_index group into one Part. Mixing families (text vs tool_call vs reasoning vs tool_result) at the same index is a plugin bug — raises ValueError instead of silently dropping content. Fallback: when no stream events were recorded (response was rehydrated from SQLite via ``from_row``), synthesize a TextPart from ``self._chunks`` plus any ``self._tool_calls`` restored by the row loader. Reasoning signatures are not recoverable from SQLite in this fallback — use ``response.to_dict()`` / ``Response.from_dict()`` for structure-preserving persistence. """ from .parts import ( ReasoningPart, TextPart, ToolCallPart, ToolResultPart, ) if not self._stream_events: # Rehydrated-from-SQLite path: assemble from _chunks + # _tool_calls so response.messages isn't empty after # from_row, and Conversation.prompt-built chains include # the assistant turn on follow-up calls. fallback_parts: list[Any] = [] text = "".join(self._chunks) if text: fallback_parts.append(TextPart(text=text)) for tc in self._tool_calls: fallback_parts.append( ToolCallPart( name=tc.name, arguments=tc.arguments or {}, tool_call_id=tc.tool_call_id, ) ) return [fallback_parts] if fallback_parts else [] # Group events by their (resolved) part_index, preserving the # order in which each index was first seen. Then build one Part # per group. This handles non-adjacent same-index events (e.g. # text → tool_call → text where the plugin pinned both text # bursts to part_index=0) by merging them into one Part. Each # group belongs to the message of its first event. groups: dict[int, list[Any]] = {} order: list[int] = [] group_message: dict[int, int] = {} for event in self._stream_events: pi = event.part_index if pi not in groups: groups[pi] = [] order.append(pi) group_message[pi] = event.message_index groups[pi].append(event) built: list[tuple[int, Any]] = [] for pi in order: evs = groups[pi] fam_first = self._event_family(evs[0].type) for e in evs: if self._event_family(e.type) != fam_first: raise ValueError( f"StreamEvent type {e.type!r} is incompatible with " f"prior type at part_index={pi}. " "Allocate a new part_index for a different content type." ) pm_merged: dict[str, Any] | None = None for e in evs: if e.provider_metadata: merged = dict(pm_merged) if pm_merged else {} for k, v in e.provider_metadata.items(): merged[k] = v pm_merged = merged mi = group_message[pi] if fam_first == "text": text = "".join(e.chunk for e in evs) if text: built.append((mi, TextPart(text=text, provider_metadata=pm_merged))) elif fam_first == "reasoning": text = "".join(e.chunk for e in evs) redacted = any(e.redacted for e in evs) if text or redacted: built.append( ( mi, ReasoningPart( text=text, redacted=redacted, provider_metadata=pm_merged, ), ) ) elif fam_first == "tool_call": tool_name = "".join(e.chunk for e in evs if e.type == "tool_call_name") args_str = "".join(e.chunk for e in evs if e.type == "tool_call_args") try: arguments = json.loads(args_str) if args_str else {} except json.JSONDecodeError: arguments = {"_raw": args_str} tool_call_id = next( (e.tool_call_id for e in evs if e.tool_call_id), None ) server_executed = any(e.server_executed for e in evs) built.append( ( mi, ToolCallPart( name=tool_name, arguments=arguments, tool_call_id=tool_call_id, server_executed=server_executed, provider_metadata=pm_merged, ), ) ) elif fam_first == "tool_result": tool_result_name = next((e.tool_name for e in evs if e.tool_name), "") tool_call_id = next( (e.tool_call_id for e in evs if e.tool_call_id), None ) server_executed = any(e.server_executed for e in evs) built.append( ( mi, ToolResultPart( name=tool_result_name, output="".join(e.chunk for e in evs), tool_call_id=tool_call_id, server_executed=server_executed, provider_metadata=pm_merged, ), ) ) # Split into per-message parts lists, message indexes in # first-seen order. message_order: list[int] = [] by_message: dict[int, list[Any]] = {} for mi, part in built: if mi not in by_message: by_message[mi] = [] message_order.append(mi) by_message[mi].append(part) messages_parts = [by_message[mi] for mi in message_order] if not messages_parts: messages_parts = [[]] # Merge in any tool calls registered via add_tool_call() that the # plugin didn't also emit as StreamEvents. Dedup by tool_call_id so # plugins using both APIs in tandem don't double-count. They join # the final message - matching the old append-at-end behavior. seen_ids = { p.tool_call_id for parts in messages_parts for p in parts if isinstance(p, ToolCallPart) and p.tool_call_id is not None } for tc in self._tool_calls: if tc.tool_call_id is not None and tc.tool_call_id in seen_ids: continue messages_parts[-1].append( ToolCallPart( name=tc.name, arguments=tc.arguments or {}, tool_call_id=tc.tool_call_id, ) ) # Hoist redacted reasoning Parts to the start of their message. # Plugins typically emit them late (when usage arrives in the # final chunk), but UIs render reasoning before content, so the # framework reorders. Relative order among redacted Parts is # preserved. for i, parts in enumerate(messages_parts): redacted_parts = [ p for p in parts if isinstance(p, ReasoningPart) and p.redacted ] if redacted_parts: other_parts = [ p for p in parts if not (isinstance(p, ReasoningPart) and p.redacted) ] messages_parts[i] = redacted_parts + other_parts return [parts for parts in messages_parts if parts] def add_tool_call(self, tool_call: ToolCall): self._tool_calls.append(_ensure_tool_call_id(tool_call)) def set_usage( self, *, input: int | None = None, output: int | None = None, details: dict | None = None, ): self.input_tokens = input self.output_tokens = output self.token_details = details def set_resolved_model(self, model_id: str): self.resolved_model = model_id @classmethod def from_row(cls, db, row, _async=False): from llm import get_async_model, get_model if _async: model = get_async_model(row["model"]) else: model = get_model(row["model"]) # Schema schema = None if row["schema_id"]: schema = json.loads(db["schemas"].get(row["schema_id"])["content"]) # Tool definitions and results for prompt tools = [ Tool( name=tool_row["name"], description=tool_row["description"], input_schema=json.loads(tool_row["input_schema"]), # In this case we don't have a reference to the actual Python code # but that's OK, we should not need it for prompts deserialized from DB implementation=None, plugin=tool_row["plugin"], ) for tool_row in db.query( """ select tools.* from tools join tool_responses on tools.id = tool_responses.tool_id where tool_responses.response_id = ? """, [row["id"]], ) ] tool_results = [ ToolResult( name=tool_results_row["name"], output=tool_results_row["output"], tool_call_id=tool_results_row["tool_call_id"], ) for tool_results_row in db.query( """ select * from tool_results where response_id = ? """, [row["id"]], ) ] all_fragments = list(db.query(FRAGMENT_SQL, {"response_id": row["id"]})) fragments = [ row["content"] for row in all_fragments if row["fragment_type"] == "prompt" ] system_fragments = [ row["content"] for row in all_fragments if row["fragment_type"] == "system" ] response = cls( model=model, prompt=Prompt( prompt=row["prompt"], model=model, fragments=fragments, attachments=[], system=row["system"], schema=schema, tools=tools, tool_results=tool_results, system_fragments=system_fragments, options=model.Options(**json.loads(row["options_json"])), ), stream=False, ) prompt_json = json.loads(row["prompt_json"] or "null") response.id = row["id"] response._prompt_json = prompt_json response.response_json = json.loads(row["response_json"] or "null") response._done = True response._chunks = [row["response"]] # Attachments response.attachments = [ Attachment.from_row(attachment_row) for attachment_row in db.query( """ select attachments.* from attachments join prompt_attachments on attachments.id = prompt_attachments.attachment_id where prompt_attachments.response_id = ? order by prompt_attachments."order" """, [row["id"]], ) ] # Tool calls response._tool_calls = [ ToolCall( name=tool_row["name"], arguments=json.loads(tool_row["arguments"]), tool_call_id=tool_row["tool_call_id"], ) for tool_row in db.query( """ select * from tool_calls where response_id = ? order by tool_call_id """, [row["id"]], ) ] return response def token_usage(self) -> str: return token_usage_string( self.input_tokens, self.output_tokens, self.token_details ) def log_to_db(self, db): # Everything - thread, turn, messages, parts, fragments, # attachments, tools - is recorded in the content-addressed # tables. The legacy tables are no longer written; they hold # history logged by older versions and are still read. # This lives here rather than in the CLI because log_to_db() # is what plugins call. from .logs import LogStore LogStore(db).log(self) def _response_to_dict(response: "_BaseResponse") -> ResponseDict: """Shared serializer for Response.to_dict / AsyncResponse.to_dict. The output is a JSON-safe dict — store it anywhere (file, Redis, Postgres, HTTP body) and round-trip via Response.from_dict or AsyncResponse.from_dict. """ options = { key: value for key, value in dict(response.prompt.options).items() if value is not None } payload: dict[str, Any] = { "model": response.model.model_id, "prompt": { "messages": [m.to_dict() for m in response.prompt.messages], }, "messages": [m.to_dict() for m in response._messages_now()], } if options: payload["prompt"]["options"] = options if response.prompt._system: payload["prompt"]["system"] = response.prompt._system # Optional audit fields — helpful for debugging, not needed for reply(). if response.id: payload["id"] = response.id if response._done: if response.input_tokens is not None or response.output_tokens is not None: usage: dict[str, Any] = {} if response.input_tokens is not None: usage["input"] = response.input_tokens if response.output_tokens is not None: usage["output"] = response.output_tokens if response.token_details is not None: usage["details"] = response.token_details payload["usage"] = usage if response._start_utcnow is not None: payload["datetime_utc"] = response._start_utcnow.isoformat() return cast(ResponseDict, payload) def _response_from_dict( data: ResponseDict, cls, *, model=None, async_: bool = False, ) -> "_BaseResponse": """Shared deserializer for Response.from_dict / AsyncResponse.from_dict.""" from .parts import Message if model is None: from llm import get_async_model, get_model getter = get_async_model if async_ else get_model model = getter(data["model"]) prompt_data = data.get("prompt", {}) input_messages = [Message.from_dict(m) for m in prompt_data.get("messages", [])] output_messages = [Message.from_dict(m) for m in data.get("messages", [])] options_kwargs = prompt_data.get("options") or {} system = prompt_data.get("system") prompt = Prompt( None, model=model, messages=input_messages, system=system, options=model.Options(**options_kwargs), ) response = cls(prompt, model=model, stream=False) # Preserve id for audit continuity. if "id" in data: response.id = data["id"] # Rebuild chunks from the assistant's text parts so response.text() # works without re-running the assembler. from .parts import TextPart response._chunks = [ p.text for m in output_messages for p in m.parts if isinstance(p, TextPart) and p.text ] # Stash the structured output so response.messages returns the # full picture (reasoning, tool calls, signatures) without needing # a StreamEvent replay. response._loaded_messages = output_messages # Rebuild _tool_calls from the restored parts so tool_calls() and # reply(tools=...) can execute serialized pending calls. Server- # executed calls stay out, matching add_tool_call() during live # streaming. from .parts import ToolCallPart response._tool_calls = [ ToolCall( name=p.name, arguments=p.arguments or {}, tool_call_id=p.tool_call_id, ) for m in output_messages for p in m.parts if isinstance(p, ToolCallPart) and not p.server_executed ] response._done = True # Restore usage if present. usage = data.get("usage") if usage: response.input_tokens = usage.get("input") response.output_tokens = usage.get("output") response.token_details = usage.get("details") return response class Response(_BaseResponse): "Sync response from a model." model: "Model" conversation: Optional["Conversation"] = None def reply( self, prompt: str | None = None, *, messages: list[Any] | None = None, tool_results: list[ToolResult] | None = None, options: dict | None = None, **kwargs, ) -> "Response": """Continue the conversation from this response. Builds the next turn's chain as ``self.prompt.messages + self.messages + [tool_message] + [user(prompt)] + messages`` and calls ``self.model.prompt(messages=chain, ...)``. If this response made tool calls and ``tool_results=`` is not passed, ``reply()`` runs ``self.execute_tool_calls()`` automatically and threads the results into the chain. Pass an explicit ``tool_results=`` list (e.g. results you mutated, or synthetic ones for testing) to skip auto-execution. """ from .parts import Message, TextPart self._force() # Forward original tools so the next turn can call them again # (mirrors Conversation.prompt's `tools or self.tools` rule). if "tools" not in kwargs and self.prompt.tools: kwargs["tools"] = self.prompt.tools if tool_results is None and self._tool_calls: tool_results = self.execute_tool_calls(tools=kwargs.get("tools")) chain: list[Any] = list(self.prompt.messages) + list(self._messages_now()) if tool_results: tool_attachments: list[Attachment] = [] for tr in tool_results: tool_attachments.extend(tr.attachments or []) _append_tool_results_to_chain(chain, tool_results, tool_attachments) if prompt: chain.append(Message(role="user", parts=[TextPart(text=prompt)])) if messages: chain.extend(messages) return self.model.prompt(messages=chain, options=options, **kwargs) def to_dict(self) -> ResponseDict: """Serialize this response for JSON persistence. Captures exactly what is needed to continue the conversation: model id, the input chain that was sent (``response.prompt.messages``), the structured assistant output (``response.messages``), and any explicit options. Pair with :meth:`Response.from_dict` to rehydrate and :meth:`Response.reply` to continue. Returns :class:`~llm.serialization.ResponseDict`. """ self._force() return _response_to_dict(self) @classmethod def from_dict( cls, data: ResponseDict, *, model: Optional["Model"] = None, ) -> "Response": """Rehydrate a Response from a ``to_dict()`` payload. The returned Response is in the ``_done`` state with ``response.text()`` and ``response.messages`` populated. ``model`` overrides the stored model id (useful for continuing on a different model). """ return cast( "Response", _response_from_dict(data, cls, model=model, async_=False) ) def on_done(self, callback): "Register a callback to be called when the response is complete." if not self._done: self.done_callbacks.append(callback) else: callback(self) def _on_done(self): for callback in self.done_callbacks: callback(self) def __str__(self) -> str: return self.text() def _force(self): if not self._done: list(self) def text(self) -> str: "Return the full text of the response, executing the prompt if needed." self._force() return "".join(self._chunks) def text_or_raise(self) -> str: return self.text() def execute_tool_calls( self, *, before_call: BeforeCallSync | None = None, after_call: AfterCallSync | None = None, tool_calls_list: list[ToolCall] | None = None, tools: list[ToolDef] | None = None, ) -> list[ToolResult]: """Execute tool calls using this response's tools. By default executes ``self.tool_calls()``; pass ``tool_calls_list=`` to execute an explicit list instead (used when resuming a chain whose history ends in unresolved calls). Pass ``tools=`` to resolve implementations from an explicit list instead of ``self.prompt.tools`` (used when a rehydrated response has pending calls but no tool implementations). """ tool_results = [] effective_tools = _wrap_tools(tools) if tools is not None else self.prompt.tools tools_by_name = { tool.name: tool for tool in effective_tools if isinstance(tool, Tool) } if tool_calls_list is None: tool_calls_list = self.tool_calls() # Run prepare() on all Toolbox instances that need it instances_to_prepare: list[Toolbox] = [] for tool_to_prep in tools_by_name.values(): inst = _get_instance(tool_to_prep.implementation) if isinstance(inst, Toolbox) and not getattr(inst, "_prepared", False): instances_to_prepare.append(inst) for inst in instances_to_prepare: inst.prepare() inst._prepared = True for tool_call in tool_calls_list: tool: Tool | None = tools_by_name.get(tool_call.name) # Tool could be None if the tool was not found in the prompt tools, # but we still call the before_call method: if before_call: try: cb_result = before_call(tool, tool_call) if inspect.isawaitable(cb_result): raise TypeError( "Asynchronous 'before_call' callback provided to a synchronous tool execution context. " "Please use an async chain/response or a synchronous callback." ) except CancelToolCall as ex: tool_results.append( ToolResult( name=tool_call.name, output="Cancelled: " + str(ex), tool_call_id=tool_call.tool_call_id, exception=ex, ) ) continue if tool is None: msg = f'tool "{tool_call.name}" does not exist' tool_results.append( ToolResult( name=tool_call.name, output="Error: " + msg, tool_call_id=tool_call.tool_call_id, exception=KeyError(msg), ) ) continue if not tool.implementation: raise ValueError( f"No implementation available for tool: {tool_call.name}" ) attachments = [] exception = None try: implementation_arguments = _implementation_arguments(tool, tool_call) if inspect.iscoroutinefunction(tool.implementation): result = asyncio.run( tool.implementation(**implementation_arguments) ) else: result = tool.implementation(**implementation_arguments) if isinstance(result, ToolOutput): attachments = result.attachments result = result.output if not isinstance(result, str): result = json.dumps(result, default=repr) except PauseChain as ex: # Pause: propagate instead of converting to an error # result. Sequential execution stops here - later calls # never started, so they can safely run on resume. ex.tool_call = tool_call ex.tool_results = list(tool_results) raise except Exception as ex: # noqa: BLE001 result = f"Error: {ex}" exception = ex tool_result_obj = ToolResult( name=tool_call.name, output=result, attachments=attachments, tool_call_id=tool_call.tool_call_id, instance=_get_instance(tool.implementation), exception=exception, ) if after_call: cb_result = after_call(tool, tool_call, tool_result_obj) if inspect.isawaitable(cb_result): raise TypeError( "Asynchronous 'after_call' callback provided to a synchronous tool execution context. " "Please use an async chain/response or a synchronous callback." ) tool_results.append(tool_result_obj) return tool_results def execute_tool_call(self, tool_call: ToolCall) -> ToolResult: "Utility method for manually executing a tool call with callbacks" tool_call = _ensure_tool_call_id(tool_call) return self.execute_tool_calls( before_call=cast(BeforeCallSync | None, self.before_call), after_call=cast(AfterCallSync | None, self.after_call), tool_calls_list=[tool_call], )[0] def tool_calls(self) -> list[ToolCall]: "Return the list of tool calls made during this response." self._force() return self._tool_calls def tool_calls_or_raise(self) -> list[ToolCall]: return self.tool_calls() def json(self) -> dict[str, Any] | None: "Return the raw JSON response from the model, if available." self._force() return self.response_json def duration_ms(self) -> int: self._force() return int(((self._end or 0) - (self._start or 0)) * 1000) def datetime_utc(self) -> str: self._force() return self._start_utcnow.isoformat() if self._start_utcnow else "" def usage(self) -> Usage: "Return token usage information for this response." self._force() return Usage( input=self.input_tokens, output=self.output_tokens, details=self.token_details, ) def _iter_events(self): """Drive self.model.execute() once and yield each raw chunk it produces. Callers normalize chunks through _process_chunk. """ if isinstance(self.model, Model): generator = self.model.execute( self.prompt, stream=self.stream, response=self, conversation=self.conversation, ) elif isinstance(self.model, KeyModel): generator = self.model.execute( self.prompt, stream=self.stream, response=self, conversation=self.conversation, key=self.model.get_key(self._key), ) else: raise TypeError("self.model must be a Model or KeyModel") for chunk in generator: assert chunk is not None yield chunk def __iter__(self) -> Iterator[str]: self._start = time.monotonic() self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) if self._done: yield from self._chunks return for chunk in self._iter_events(): text = self._process_chunk(chunk) if text is not None: yield text if self.conversation: self.conversation._record_response(self) self._end = time.monotonic() self._done = True self._on_done() def stream_events(self): """Yield StreamEvent objects as the model produces them. Whichever of __iter__ and stream_events runs first during live streaming consumes the underlying generator. After completion, both work — each replays from its own buffer. """ if self._done: yield from self._stream_events return self._start = time.monotonic() self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) for chunk in self._iter_events(): # _process_chunk appends to self._stream_events; use it as # the canonical source for what to yield so the replay path # matches the live path byte-for-byte. self._process_chunk(chunk) yield self._stream_events[-1] if self.conversation: self.conversation._record_response(self) self._end = time.monotonic() self._done = True self._on_done() def messages(self) -> list[Any]: """List of Message objects produced by this response. Almost always a single assistant Message; multiple messages are possible for providers that emit multi-message responses during server-side tool execution. Forces execution if the response has not yet been drained, so ``response.messages()`` is safe to call without a prior ``response.text()`` / iteration. Responses rehydrated via ``Response.from_dict`` short-circuit and return the stored messages directly. """ self._force() return self._messages_now() def __repr__(self): text = "... not yet done ..." if self._done: text = "".join(self._chunks) return f"" class AsyncResponse(_BaseResponse): "Async response from a model." model: "AsyncModel" conversation: Optional["AsyncConversation"] = None async def reply( self, prompt: str | None = None, *, messages: list[Any] | None = None, tool_results: list[ToolResult] | None = None, options: dict | None = None, **kwargs, ) -> "AsyncResponse": """Async counterpart of Response.reply(). Requires this response to have been awaited (so self.messages is available). Awaitable so the auto-execute path can ``await self.execute_tool_calls()``. See ``Response.reply`` for the ``tool_results=`` semantics. """ from .parts import Message, TextPart if not self._done: raise ValueError( "Response not yet awaited — call `await response` before reply()" ) if "tools" not in kwargs and self.prompt.tools: kwargs["tools"] = self.prompt.tools if tool_results is None and self._tool_calls: tool_results = await self.execute_tool_calls(tools=kwargs.get("tools")) chain: list[Any] = list(self.prompt.messages) + list(self._messages_now()) if tool_results: tool_attachments: list[Attachment] = [] for tr in tool_results: tool_attachments.extend(tr.attachments or []) _append_tool_results_to_chain(chain, tool_results, tool_attachments) if prompt: chain.append(Message(role="user", parts=[TextPart(text=prompt)])) if messages: chain.extend(messages) return self.model.prompt(messages=chain, options=options, **kwargs) def to_dict(self) -> ResponseDict: """Async counterpart of Response.to_dict(). Requires awaiting.""" if not self._done: raise ValueError( "Response not yet awaited — call `await response` before to_dict()" ) return _response_to_dict(self) @classmethod def from_dict( cls, data: ResponseDict, *, model: Optional["AsyncModel"] = None, ) -> "AsyncResponse": """Async counterpart of Response.from_dict().""" return cast( "AsyncResponse", _response_from_dict(data, cls, model=model, async_=True) ) @classmethod def from_row(cls, db, row, _async=False): return super().from_row(db, row, _async=True) async def on_done(self, callback): "Register a callback to be called when the response is complete." if not self._done: self.done_callbacks.append(callback) else: if callable(callback): # Ensure we handle both sync and async callbacks correctly processed_callback = callback(self) if inspect.isawaitable(processed_callback): await processed_callback elif inspect.isawaitable(callback): await callback async def _on_done(self): for callback_func in self.done_callbacks: if callable(callback_func): processed_callback = callback_func(self) if inspect.isawaitable(processed_callback): await processed_callback elif inspect.isawaitable(callback_func): await callback_func async def execute_tool_calls( self, *, before_call: BeforeCallAsync | None = None, after_call: AfterCallAsync | None = None, tool_calls_list: list[ToolCall] | None = None, tools: list[ToolDef] | None = None, ) -> list[ToolResult]: """Execute tool calls using this response's tools. By default executes ``await self.tool_calls()``; pass ``tool_calls_list=`` to execute an explicit list instead (used when resuming a chain whose history ends in unresolved calls). Pass ``tools=`` to resolve implementations from an explicit list instead of ``self.prompt.tools`` (used when a rehydrated response has pending calls but no tool implementations). """ if tool_calls_list is None: tool_calls_list = await self.tool_calls() effective_tools = _wrap_tools(tools) if tools is not None else self.prompt.tools tools_by_name = { tool.name: tool for tool in effective_tools if isinstance(tool, Tool) } # Run async prepare_async() on all Toolbox instances that need it instances_to_prepare: list[Toolbox] = [] for tool_to_prep in tools_by_name.values(): inst = _get_instance(tool_to_prep.implementation) if isinstance(inst, Toolbox) and not getattr( inst, "_async_prepared", False ): instances_to_prepare.append(inst) for inst in instances_to_prepare: await inst.prepare_async() inst._async_prepared = True indexed_results: list[tuple[int, ToolResult]] = [] async_tasks: list[asyncio.Task] = [] async_task_indexes: list[int] = [] # Defined failure semantics: a pause or error in one call must not # orphan concurrently-running siblings. Pauses and hook failures # are collected here and raised only after every task that was # started has finished. paused: list[tuple[int, PauseChain]] = [] failures: list[tuple[int, BaseException]] = [] for idx, tc in enumerate(tool_calls_list): tool: Tool | None = tools_by_name.get(tc.name) exception: Exception | None = None if tool is None or not tool.implementation: # Mirror the sync executor: append an error ToolResult so # the provider still receives a result for every tool # call. before_call fires even though the tool is # unavailable. if before_call: try: cb = before_call(tool, tc) if inspect.isawaitable(cb): await cb except CancelToolCall as ex: indexed_results.append( ( idx, ToolResult( name=tc.name, output="Cancelled: " + str(ex), tool_call_id=tc.tool_call_id, exception=ex, ), ) ) continue except Exception as ex: # noqa: BLE001 failures.append((idx, ex)) break reason = "does not exist" if tool is None else "has no implementation" msg = f'tool "{tc.name}" {reason}' indexed_results.append( ( idx, ToolResult( name=tc.name, output="Error: " + msg, tool_call_id=tc.tool_call_id, exception=KeyError(msg), ), ) ) continue if inspect.iscoroutinefunction(tool.implementation): async def run_async(tc=tc, tool=tool, idx=idx): # before_call inside the task if before_call: try: cb = before_call(tool, tc) if inspect.isawaitable(cb): await cb except CancelToolCall as ex: return idx, ToolResult( name=tc.name, output="Cancelled: " + str(ex), tool_call_id=tc.tool_call_id, exception=ex, ) exception = None attachments = [] try: result = await tool.implementation( **_implementation_arguments(tool, tc) ) if isinstance(result, ToolOutput): attachments.extend(result.attachments) result = result.output output = ( result if isinstance(result, str) else json.dumps(result, default=repr) ) except PauseChain as ex: # Propagates out of the task; collected after # the gather so siblings finish first. ex.tool_call = tc raise except Exception as ex: # noqa: BLE001 output = f"Error: {ex}" exception = ex tr = ToolResult( name=tc.name, output=output, attachments=attachments, tool_call_id=tc.tool_call_id, instance=_get_instance(tool.implementation), exception=exception, ) # after_call inside the task if tool is not None and after_call: cb2 = after_call(tool, tc, tr) if inspect.isawaitable(cb2): await cb2 return idx, tr async_tasks.append(asyncio.create_task(run_async())) async_task_indexes.append(idx) else: # Sync implementation: do hooks and call inline if before_call: try: cb = before_call(tool, tc) if inspect.isawaitable(cb): await cb except CancelToolCall as ex: indexed_results.append( ( idx, ToolResult( name=tc.name, output="Cancelled: " + str(ex), tool_call_id=tc.tool_call_id, exception=ex, ), ) ) continue except Exception as ex: # noqa: BLE001 failures.append((idx, ex)) break exception = None attachments = [] try: res = tool.implementation(**_implementation_arguments(tool, tc)) if inspect.isawaitable(res): res = await res if isinstance(res, ToolOutput): attachments.extend(res.attachments) res = res.output output = ( res if isinstance(res, str) else json.dumps(res, default=repr) ) except PauseChain as ex: # Inline execution stops here; later calls never # start. Tasks already started are still awaited # below before the pause propagates. ex.tool_call = tc paused.append((idx, ex)) break except Exception as ex: # noqa: BLE001 output = f"Error: {ex}" exception = ex tr = ToolResult( name=tc.name, output=output, attachments=attachments, tool_call_id=tc.tool_call_id, instance=_get_instance(tool.implementation), exception=exception, ) try: if after_call: cb2 = after_call(tool, tc, tr) if inspect.isawaitable(cb2): await cb2 except Exception as ex: # noqa: BLE001 failures.append((idx, ex)) break indexed_results.append((idx, tr)) # Await every task that was started; return_exceptions so a pause # or hook failure in one task cannot orphan its siblings mid-flight. if async_tasks: outcomes = await asyncio.gather(*async_tasks, return_exceptions=True) for task_idx, outcome in zip(async_task_indexes, outcomes): if isinstance(outcome, PauseChain): paused.append((task_idx, outcome)) elif isinstance(outcome, BaseException): failures.append((task_idx, outcome)) else: indexed_results.append(outcome) # Reorder by original index indexed_results.sort(key=lambda x: x[0]) results = [tr for _, tr in indexed_results] # Hook failures are bugs: raise the first by call order. if failures: failures.sort(key=lambda item: item[0]) raise failures[0][1] # Pauses propagate with the completed sibling results attached. if paused: paused.sort(key=lambda item: item[0]) pause = paused[0][1] pause.tool_results = results raise pause return results async def execute_tool_call(self, tool_call: ToolCall) -> ToolResult: "Asynchronous counterpart to :meth:`Response.execute_tool_call`." tool_call = _ensure_tool_call_id(tool_call) results = await self.execute_tool_calls( before_call=cast(BeforeCallAsync | None, self.before_call), after_call=cast(AfterCallAsync | None, self.after_call), tool_calls_list=[tool_call], ) return results[0] def __aiter__(self): self._start = time.monotonic() self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) if self._done: self._iter_chunks = list(self._chunks) # Make a copy for iteration return self def _ensure_async_generator(self): if not hasattr(self, "_generator"): if isinstance(self.model, AsyncModel): self._generator = self.model.execute( self.prompt, stream=self.stream, response=self, conversation=self.conversation, ) elif isinstance(self.model, AsyncKeyModel): self._generator = self.model.execute( self.prompt, stream=self.stream, response=self, conversation=self.conversation, key=self.model.get_key(self._key), ) else: raise ValueError("self.model must be an AsyncModel or AsyncKeyModel") async def _async_finalize(self): if self.conversation: self.conversation._record_response(self) self._end = time.monotonic() self._done = True if hasattr(self, "_generator"): del self._generator await self._on_done() async def __anext__(self) -> str: if self._done: if hasattr(self, "_iter_chunks") and self._iter_chunks: return self._iter_chunks.pop(0) raise StopAsyncIteration self._ensure_async_generator() # Skip non-text events — iteration yields only text. Loop until # we find a text chunk or the generator is exhausted. while True: try: chunk = await self._generator.__anext__() except StopAsyncIteration: await self._async_finalize() raise assert chunk is not None text = self._process_chunk(chunk) if text is not None: return text async def astream_events(self): """Yield StreamEvent objects as the model produces them (async).""" if self._done: for event in self._stream_events: yield event return self._start = time.monotonic() self._start_utcnow = datetime.datetime.now(datetime.timezone.utc) self._ensure_async_generator() try: while True: try: chunk = await self._generator.__anext__() except StopAsyncIteration: await self._async_finalize() return assert chunk is not None self._process_chunk(chunk) yield self._stream_events[-1] finally: pass async def messages(self) -> list[Any]: """List of Message objects produced by this response. Awaits ``self._force()`` so ``await response.messages()`` is safe to call without first awaiting ``response.text()`` or iterating the stream. Responses rehydrated via ``AsyncResponse.from_dict`` short-circuit and return the stored messages. """ await self._force() return self._messages_now() async def _force(self): if not self._done: temp_chunks = [] async for chunk in self: temp_chunks.append(chunk) # This should populate self._chunks return self def text_or_raise(self) -> str: if not self._done: raise ValueError("Response not yet awaited") return "".join(self._chunks) async def text(self) -> str: "Return the full text of the response, executing the prompt if needed." await self._force() return "".join(self._chunks) async def tool_calls(self) -> list[ToolCall]: "Return the list of tool calls made during this response." await self._force() return self._tool_calls def tool_calls_or_raise(self) -> list[ToolCall]: if not self._done: raise ValueError("Response not yet awaited") return self._tool_calls async def json(self) -> dict[str, Any] | None: "Return the raw JSON response from the model, if available." await self._force() return self.response_json async def duration_ms(self) -> int: await self._force() return int(((self._end or 0) - (self._start or 0)) * 1000) async def datetime_utc(self) -> str: await self._force() return self._start_utcnow.isoformat() if self._start_utcnow else "" async def usage(self) -> Usage: "Return token usage information for this response." await self._force() return Usage( input=self.input_tokens, output=self.output_tokens, details=self.token_details, ) def __await__(self): return self._force().__await__() async def to_sync_response(self) -> Response: await self._force() # This conversion might be tricky if the model is AsyncModel, # as Response expects a sync Model. For simplicity, we'll assume # the primary use case is data transfer after completion. # The model type on the new Response might need careful handling # if it's intended for further execution. # For now, let's assume self.model can be cast or is compatible. sync_model = self.model if not isinstance(self.model, (Model, KeyModel)): # This is a placeholder. A proper conversion or shared base might be needed # if the sync_response needs to be fully functional with its model. # For now, we pass the async model, which might limit what sync_response can do. pass response = Response( self.prompt, sync_model, # This might need adjustment based on how Model/AsyncModel relate self.stream, # conversation type needs to be compatible too. conversation=( self.conversation.to_sync_conversation() if self.conversation else None ), ) response.id = self.id response._chunks = list(self._chunks) # Copy chunks response._done = self._done response._end = self._end response._start = self._start response._start_utcnow = self._start_utcnow response.input_tokens = self.input_tokens response.output_tokens = self.output_tokens response.token_details = self.token_details response._prompt_json = self._prompt_json response.response_json = self.response_json response._tool_calls = list(self._tool_calls) # Without these the sync response falls back to assembling a bare # TextPart from _chunks, so reasoning, redacted markers and every # part's provider_metadata are lost. The CLI converts before # logging, so that loss would apply to every async response. response._stream_events = list(self._stream_events) response.attachments = list(self.attachments) response.resolved_model = self.resolved_model return response @classmethod def fake( cls, model: "AsyncModel", prompt: str, *attachments: list[Attachment], system: str, response: str, ): "Utility method to help with writing tests" response_obj = cls( model=model, prompt=Prompt( prompt, model=model, attachments=attachments, system=system, ), stream=False, ) response_obj._done = True response_obj._chunks = [response] return response_obj def __repr__(self): text = "... not yet awaited ..." if self._done: text = "".join(self._chunks) return f"" def _append_tool_results_to_chain(chain, tool_results, attachments) -> list[Any]: """Append a tool-role message carrying ToolResults to a message chain, plus a trailing user-role message for any attachments the tools returned (mimics the legacy attachments=[] kwarg behavior).""" from .parts import ( AttachmentPart, Message, ToolResultPart, ) if tool_results: chain.append( Message( role="tool", parts=[ ToolResultPart( name=tr.name, output=tr.output, tool_call_id=tr.tool_call_id, exception=_format_tool_exception(tr.exception), ) for tr in tool_results ], ) ) if attachments: chain.append( Message( role="user", parts=[AttachmentPart(attachment=a) for a in attachments], ) ) return chain def _chain_for_tool_results(prior_response, tool_results, attachments) -> list[Any]: """Build the message chain for a tool-result turn in a chain loop. Takes the prior response's full input chain + its structured output, then appends a tool-role message carrying the new ToolResult outputs. This is what gives ``response.prompt.messages`` on the tool- result turn the complete history for the next provider call — including any reasoning signatures or thoughtSignatures from the prior turn. """ chain: list[Any] = list(prior_response.prompt.messages) + list( prior_response._messages_now() ) return _append_tool_results_to_chain(chain, tool_results, attachments) def _trailing_pending_tool_calls(messages) -> list[ToolCall]: """Find unresolved tool calls at the end of a message history. Returns ToolCall objects from the last assistant message containing locally-executable tool_call parts, minus any that already have a matching tool_result in subsequent tool-role messages. Returns [] when the history has moved on past those calls (a user/assistant/ system message follows them) - resuming only makes sense when the calls are the latest thing that happened. Matching uses tool_call_id when present; id-less calls (histories persisted before ids were guaranteed) match results by name, one result consumed per call. """ from .parts import ToolCallPart, ToolResultPart last_index = None call_parts: list[Any] = [] for i, msg in enumerate(messages or []): parts = getattr(msg, "parts", None) or [] calls = [ p for p in parts if isinstance(p, ToolCallPart) and not p.server_executed ] if getattr(msg, "role", None) == "assistant" and calls: last_index = i call_parts = calls if last_index is None: return [] results: list[Any] = [] for msg in messages[last_index + 1 :]: role = getattr(msg, "role", None) if role == "tool": results.extend( p for p in (getattr(msg, "parts", None) or []) if isinstance(p, ToolResultPart) ) else: # Conversation moved on past these calls return [] matched_ids = {r.tool_call_id for r in results if r.tool_call_id} unmatched_names = [r.name for r in results if not r.tool_call_id] pending = [] for part in call_parts: if part.tool_call_id: if part.tool_call_id in matched_ids: continue elif part.name in unmatched_names: unmatched_names.remove(part.name) continue pending.append( ToolCall( name=part.name, arguments=part.arguments or {}, tool_call_id=part.tool_call_id, ) ) return pending class _BaseChainResponse: prompt: "Prompt" stream: bool conversation: Optional["_BaseConversation"] = None _key: str | None = None def __init__( self, prompt: Prompt, model: "_BaseModel", stream: bool, conversation: _BaseConversation, key: str | None = None, chain_limit: int | None = 10, before_call: BeforeCallSync | BeforeCallAsync | None = None, after_call: AfterCallSync | AfterCallAsync | None = None, ): self.prompt = prompt self.model = model self.stream = stream self._key = key self._responses: list[Any] = [] self.conversation = conversation self.chain_limit = chain_limit self.before_call = before_call self.after_call = after_call def log_to_db(self, db): for response in self._responses: if isinstance(response, AsyncResponse): sync_response = asyncio.run(response.to_sync_response()) elif isinstance(response, Response): sync_response = response else: assert False, "Should have been a Response or AsyncResponse" sync_response.log_to_db(db) def _pending_tool_calls(self) -> list[ToolCall]: """Unresolved tool calls at the end of this chain's history. Non-empty when the supplied messages= end in an assistant message whose tool calls have no results yet - e.g. a chain that paused on PauseChain and is being resumed from persisted history.""" if not self.prompt.tools: return [] return _trailing_pending_tool_calls(self.prompt.messages) def _resume_prompt(self, tool_results: list[ToolResult]) -> Prompt: """The first prompt for a resumed chain: the original history plus a tool-role message carrying the freshly-executed results - the same shape as the chain loop's own tool-result turns.""" prompt = self.prompt attachments = [] for tool_result in tool_results: attachments.extend(tool_result.attachments) next_chain = _append_tool_results_to_chain( list(prompt.messages), tool_results, attachments ) return Prompt( "", self.model, tools=prompt.tools, tool_results=tool_results, messages=next_chain, system=prompt._system, system_fragments=prompt.system_fragments, options=prompt.options, attachments=attachments, hide_reasoning=prompt.hide_reasoning, ) class ChainResponse(_BaseChainResponse): _responses: list["Response"] before_call: BeforeCallSync | None = None after_call: AfterCallSync | None = None def responses(self) -> Iterator[Response]: prompt = self.prompt count = 0 initial_response = Response( prompt, self.model, self.stream, key=self._key, conversation=self.conversation, before_call=self.before_call, after_call=self.after_call, ) # Resume: a history ending in unresolved tool calls means a # previous run stopped (paused or crashed) before executing # them. Execute those calls first - through the normal # before_call/after_call machinery - then start the loop on # the tool-result turn. This could raise llm.PauseChain. pending_tool_calls = self._pending_tool_calls() if pending_tool_calls: tool_results = initial_response.execute_tool_calls( before_call=self.before_call, after_call=self.after_call, tool_calls_list=pending_tool_calls, ) initial_response = Response( self._resume_prompt(tool_results), self.model, self.stream, key=self._key, conversation=self.conversation, before_call=self.before_call, after_call=self.after_call, ) current_response: Response | None = initial_response while current_response: count += 1 yield current_response self._responses.append(current_response) if self.chain_limit and count >= self.chain_limit: raise ValueError(f"Chain limit of {self.chain_limit} exceeded.") # This could raise llm.CancelToolCall: tool_results = current_response.execute_tool_calls( before_call=self.before_call, after_call=self.after_call ) attachments = [] for tool_result in tool_results: attachments.extend(tool_result.attachments) if tool_results: # Pre-bake the full chain for the tool-result turn so # response.prompt.messages is what gets sent — carries # thoughtSignatures, thinking signatures, and everything # else the model needs for the next call. next_chain = _chain_for_tool_results( current_response, tool_results, attachments ) current_response = Response( Prompt( "", # Next prompt text is empty; tool_results drive it self.model, tools=current_response.prompt.tools, tool_results=tool_results, messages=next_chain, # Carry system + system_fragments forward so # stateless-per-turn adapters (OpenAI and # friends that read prompt.system directly) # keep seeing the system prompt on every call # of the chain loop. system=self.prompt._system, system_fragments=self.prompt.system_fragments, options=self.prompt.options, attachments=attachments, hide_reasoning=current_response.prompt.hide_reasoning, ), self.model, stream=self.stream, key=self._key, conversation=self.conversation, before_call=self.before_call, after_call=self.after_call, ) else: current_response = None break def __iter__(self) -> Iterator[str]: # Rounds of a chain are separate model responses; joined with # nothing between them the text of one runs straight into the # next ("...have dragons.Now that I..."). Yield one space at # each boundary where neither side brings its own whitespace. # Display only: the separator never enters any response's # recorded events, so it is not stored or hashed. last_char = "" for response_item in self.responses(): first_chunk = True for chunk in response_item: if not chunk: continue if ( first_chunk and last_char and not last_char.isspace() and not chunk[0].isspace() ): yield " " first_chunk = False yield chunk last_char = chunk[-1] def stream_events(self): "Yield StreamEvents from every response in the chain." from .parts import StreamEvent # The same round-boundary separator as __iter__, synthesized at # the chain level so it is never part of a response's events. last_char = "" for response_item in self.responses(): first_text = True for event in response_item.stream_events(): if event.type == "text" and event.chunk: if ( first_text and last_char and not last_char.isspace() and not event.chunk[0].isspace() ): yield StreamEvent(type="text", chunk=" ") first_text = False last_char = event.chunk[-1] yield event def text(self) -> str: return "".join(self) class AsyncChainResponse(_BaseChainResponse): _responses: list["AsyncResponse"] before_call: BeforeCallAsync | None = None after_call: AfterCallAsync | None = None async def responses(self) -> AsyncIterator[AsyncResponse]: prompt = self.prompt count = 0 initial_response = AsyncResponse( prompt, self.model, self.stream, key=self._key, conversation=self.conversation, before_call=self.before_call, after_call=self.after_call, ) # Resume: see ChainResponse.responses() - execute trailing # unresolved tool calls before the first provider call. This # could raise llm.PauseChain. pending_tool_calls = self._pending_tool_calls() if pending_tool_calls: tool_results = await initial_response.execute_tool_calls( before_call=self.before_call, after_call=self.after_call, tool_calls_list=pending_tool_calls, ) initial_response = AsyncResponse( self._resume_prompt(tool_results), self.model, self.stream, key=self._key, conversation=self.conversation, before_call=self.before_call, after_call=self.after_call, ) current_response: AsyncResponse | None = initial_response while current_response: count += 1 yield current_response self._responses.append(current_response) if self.chain_limit and count >= self.chain_limit: raise ValueError(f"Chain limit of {self.chain_limit} exceeded.") # This could raise llm.CancelToolCall: tool_results = await current_response.execute_tool_calls( before_call=self.before_call, after_call=self.after_call ) if tool_results: attachments = [] for tool_result in tool_results: attachments.extend(tool_result.attachments) # Pre-bake chain so prompt.messages carries full history # + any thinking/tool-call signatures from prior turn. next_chain = _chain_for_tool_results( current_response, tool_results, attachments ) prompt = Prompt( "", self.model, tools=current_response.prompt.tools, tool_results=tool_results, messages=next_chain, # Carry system + system_fragments forward — same # reasoning as the sync path. system=self.prompt._system, system_fragments=self.prompt.system_fragments, options=self.prompt.options, attachments=attachments, hide_reasoning=current_response.prompt.hide_reasoning, ) current_response = AsyncResponse( prompt, self.model, stream=self.stream, key=self._key, conversation=self.conversation, before_call=self.before_call, after_call=self.after_call, ) else: current_response = None break async def __aiter__(self) -> AsyncIterator[str]: # Round-boundary separator - same reasoning as the sync chain. last_char = "" async for response_item in self.responses(): first_chunk = True async for chunk in response_item: if not chunk: continue if ( first_chunk and last_char and not last_char.isspace() and not chunk[0].isspace() ): yield " " first_chunk = False yield chunk last_char = chunk[-1] async def astream_events(self): "Yield StreamEvents from every response in the chain." from .parts import StreamEvent # Same round-boundary separator as __aiter__, synthesized at the # chain level so it is never part of a response's events. last_char = "" async for response_item in self.responses(): first_text = True async for event in response_item.astream_events(): if event.type == "text" and event.chunk: if ( first_text and last_char and not last_char.isspace() and not event.chunk[0].isspace() ): yield StreamEvent(type="text", chunk=" ") first_text = False last_char = event.chunk[-1] yield event async def text(self) -> str: all_chunks = [] async for chunk in self: all_chunks.append(chunk) return "".join(all_chunks) class Options(BaseModel): model_config = ConfigDict(extra="forbid") _Options = Options class _get_key_mixin: needs_key: str | None = None key: str | None = None key_env_var: str | None = None def get_key(self, explicit_key: str | None = None) -> str | None: from llm import get_key if self.needs_key is None: # This model doesn't use an API key return None if self.key is not None: # Someone already set model.key='...' return self.key # Attempt to load a key using llm.get_key() key_value = get_key( explicit_key=explicit_key, key_alias=self.needs_key, env_var=self.key_env_var, ) if key_value: return key_value # Show a useful error message message = f"No key found - add one using 'llm keys set {self.needs_key}'" if self.key_env_var: message += f" or set the {self.key_env_var} environment variable" raise NeedsKeyException(message) class _BaseModel(ABC, _get_key_mixin): model_id: str can_stream: bool = False attachment_types: set[str] | frozenset[str] = frozenset() supports_schema = False supports_tools = False @property def supported_server_side_tools(self) -> tuple[type[ServerSideTool], ...]: """Server-side tool classes accepted by this model instance.""" return () class Options(_Options): pass def _validate_attachments( self, attachments: list[Attachment] | None = None ) -> None: if attachments and not self.attachment_types: raise ValueError("This model does not support attachments") for attachment in attachments or []: attachment_type = attachment.resolve_type() if attachment_type not in self.attachment_types: raise ValueError( f"This model does not support attachments of type '{attachment_type}', " f"only {', '.join(self.attachment_types)}" ) def __str__(self) -> str: return "{}{}: {}".format( self.__class__.__name__, " (async)" if isinstance(self, (AsyncModel, AsyncKeyModel)) else "", self.model_id, ) def __repr__(self) -> str: return f"<{self!s}>" class _Model(_BaseModel): def conversation( self, tools: list[ToolDef] | None = None, before_call: BeforeCallSync | None = None, after_call: AfterCallSync | None = None, chain_limit: int | None = None, ) -> Conversation: return Conversation( model=self, tools=tools, before_call=before_call, after_call=after_call, chain_limit=chain_limit, ) def prompt( self, prompt: str | None = None, *, fragments: list[str | Fragment] | None = None, attachments: list[Attachment] | None = None, system: str | None = None, system_fragments: list[str | Fragment] | None = None, messages: list[Any] | None = None, stream: bool = True, schema: dict | type[BaseModel] | None = None, tools: list[ToolDef] | None = None, tool_results: list[ToolResult] | None = None, options: dict | None = None, hide_reasoning: bool = False, **kwargs, ) -> Response: key_value = kwargs.pop("key", None) merged = _merge_options(options, kwargs) self._validate_attachments(attachments) if messages is not None: # messages= is the authoritative history; the other prompt # arguments are this turn's new input, folded in so that # response.prompt.messages stays exactly what the model sees. messages = _append_turn_input( list(messages), prompt, fragments, attachments, tool_results ) return Response( Prompt( prompt, fragments=fragments, attachments=attachments, system=system, schema=schema, tools=tools, tool_results=tool_results, system_fragments=system_fragments, messages=messages, model=self, options=self.Options(**merged), hide_reasoning=hide_reasoning, ), self, stream, key=key_value, ) def chain( self, prompt: str | None = None, *, fragments: list[str] | None = None, attachments: list[Attachment] | None = None, system: str | None = None, system_fragments: list[str] | None = None, messages: list[Any] | None = None, stream: bool = True, schema: dict | type[BaseModel] | None = None, tools: list[ToolDef] | None = None, tool_results: list[ToolResult] | None = None, before_call: BeforeCallSync | None = None, after_call: AfterCallSync | None = None, key: str | None = None, options: dict | None = None, hide_reasoning: bool = False, ) -> ChainResponse: return self.conversation().chain( prompt=prompt, fragments=fragments, attachments=attachments, system=system, system_fragments=system_fragments, messages=messages, stream=stream, schema=schema, tools=tools, tool_results=tool_results, before_call=before_call, after_call=after_call, key=key, options=options, hide_reasoning=hide_reasoning, ) class Model(_Model): @abstractmethod def execute( self, prompt: Prompt, stream: bool, response: Response, conversation: Conversation | None, ) -> Iterator[Union[str, "StreamEvent"]]: pass class KeyModel(_Model): @abstractmethod def execute( self, prompt: Prompt, stream: bool, response: Response, conversation: Conversation | None, key: str | None, ) -> Iterator[Union[str, "StreamEvent"]]: pass class _AsyncModel(_BaseModel): def conversation( self, tools: list[ToolDef] | None = None, before_call: BeforeCallAsync | None = None, after_call: AfterCallAsync | None = None, chain_limit: int | None = None, ) -> AsyncConversation: return AsyncConversation( model=self, tools=tools, before_call=before_call, after_call=after_call, chain_limit=chain_limit, ) def prompt( self, prompt: str | None = None, *, fragments: list[str | Fragment] | None = None, attachments: list[Attachment] | None = None, system: str | None = None, schema: dict | type[BaseModel] | None = None, tools: list[ToolDef] | None = None, tool_results: list[ToolResult] | None = None, system_fragments: list[str | Fragment] | None = None, messages: list[Any] | None = None, stream: bool = True, options: dict | None = None, hide_reasoning: bool = False, **kwargs, ) -> AsyncResponse: key_value = kwargs.pop("key", None) merged = _merge_options(options, kwargs) self._validate_attachments(attachments) if messages is not None: # Same fold as Model.prompt: messages= is the history, the # other prompt arguments are this turn's new input. messages = _append_turn_input( list(messages), prompt, fragments, attachments, tool_results ) return AsyncResponse( Prompt( prompt, fragments=fragments, attachments=attachments, system=system, schema=schema, tools=tools, tool_results=tool_results, system_fragments=system_fragments, messages=messages, model=self, options=self.Options(**merged), hide_reasoning=hide_reasoning, ), self, stream, key=key_value, ) def chain( self, prompt: str | None = None, *, fragments: list[str] | None = None, attachments: list[Attachment] | None = None, system: str | None = None, system_fragments: list[str] | None = None, messages: list[Any] | None = None, stream: bool = True, schema: dict | type[BaseModel] | None = None, tools: list[ToolDef] | None = None, tool_results: list[ToolResult] | None = None, before_call: BeforeCallAsync | None = None, after_call: AfterCallAsync | None = None, key: str | None = None, options: dict | None = None, hide_reasoning: bool = False, ) -> AsyncChainResponse: return self.conversation().chain( prompt=prompt, fragments=fragments, attachments=attachments, system=system, system_fragments=system_fragments, messages=messages, stream=stream, schema=schema, tools=tools, tool_results=tool_results, before_call=before_call, after_call=after_call, key=key, options=options, hide_reasoning=hide_reasoning, ) class AsyncModel(_AsyncModel): @abstractmethod async def execute( self, prompt: Prompt, stream: bool, response: AsyncResponse, conversation: AsyncConversation | None, ) -> AsyncGenerator[Union[str, "StreamEvent"], None]: if False: # Ensure it's a generator type yield "" class AsyncKeyModel(_AsyncModel): @abstractmethod async def execute( self, prompt: Prompt, stream: bool, response: AsyncResponse, conversation: AsyncConversation | None, key: str | None, ) -> AsyncGenerator[Union[str, "StreamEvent"], None]: if False: # Ensure it's a generator type yield "" class EmbeddingModel(ABC, _get_key_mixin): model_id: str key: str | None = None needs_key: str | None = None key_env_var: str | None = None supports_text: bool = True supports_binary: bool = False batch_size: int | None = None def _check(self, item: str | bytes): if not self.supports_binary and isinstance(item, bytes): raise ValueError( "This model does not support binary data, only text strings" ) if not self.supports_text and isinstance(item, str): raise ValueError( "This model does not support text strings, only binary data" ) def embed(self, item: str | bytes) -> list[float]: "Embed a single text string or binary blob, return a list of floats" self._check(item) return next(iter(self.embed_batch([item]))) def embed_multi( self, items: Iterable[str | bytes], batch_size: int | None = None ) -> Iterator[list[float]]: "Embed multiple items in batches according to the model batch_size" iter_items = iter(items) effective_batch_size = self.batch_size if batch_size is None else batch_size if (not self.supports_binary) or (not self.supports_text): def checking_iter(inner_items): for item_to_check in inner_items: self._check(item_to_check) yield item_to_check iter_items = checking_iter(items) if effective_batch_size is None: yield from self.embed_batch(iter_items) return while True: batch_items = list(islice(iter_items, effective_batch_size)) if not batch_items: break yield from self.embed_batch(batch_items) @abstractmethod def embed_batch(self, items: Iterable[str | bytes]) -> Iterator[list[float]]: """ Embed a batch of strings or blobs, return a list of lists of floats """ def __str__(self) -> str: return f"{self.__class__.__name__}: {self.model_id}" def __repr__(self) -> str: return f"<{self!s}>" @dataclass class ModelWithAliases: "A model with its optional async counterpart and aliases." model: Model async_model: AsyncModel aliases: set[str] def matches(self, query: str) -> bool: query_lower = query.lower() all_strings: list[str] = [] all_strings.extend(self.aliases) if self.model: all_strings.append(str(self.model)) if self.async_model: all_strings.append(str(self.async_model.model_id)) return any(query_lower in alias.lower() for alias in all_strings) @dataclass class EmbeddingModelWithAliases: model: EmbeddingModel aliases: set[str] def matches(self, query: str) -> bool: query_lower = query.lower() all_strings: list[str] = [] all_strings.extend(self.aliases) all_strings.append(str(self.model)) return any(query_lower in alias.lower() for alias in all_strings) def _format_tool_exception(exception) -> str | None: """Render a tool's exception the way it is recorded. ToolResult carries the exception object; ToolResultPart carries the rendered string, so the chain has to convert rather than drop it. """ if exception is None: return None if isinstance(exception, str): return exception return f"{exception.__class__.__name__}: {exception!s}" def _conversation_name(text): # Collapse whitespace, including newlines text = re.sub(r"\s+", " ", text) if len(text) <= CONVERSATION_NAME_LENGTH: return text return text[: CONVERSATION_NAME_LENGTH - 1] + "…" def _ensure_dict_schema(schema): """Convert a Pydantic model to a JSON schema dict if needed.""" if schema and not isinstance(schema, dict) and issubclass(schema, BaseModel): schema_dict = schema.model_json_schema() _remove_titles_recursively(schema_dict) return schema_dict return schema def _remove_titles_recursively(obj): """Recursively remove all 'title' fields from a nested dictionary.""" if isinstance(obj, dict): # Remove title if present obj.pop("title", None) # Recursively process all values for value in obj.values(): _remove_titles_recursively(value) elif isinstance(obj, list): # Process each item in lists for item in obj: _remove_titles_recursively(item) def _get_instance(implementation): if hasattr(implementation, "__self__"): return implementation.__self__ return None ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/parts.py0000644000175100017510000002721115234420023014155 0ustar00runnerrunner"""Part, Message, and StreamEvent value types. Parts represent the structured content of model interactions: text, reasoning, tool calls, tool results, and attachments. A Message wraps a list of Parts with a role. StreamEvent wraps a streaming chunk with type information so consumers can distinguish text from reasoning from tool call fragments as they arrive. These types are pure values — identity (ids, parent links, storage keys) is a storage concern that lives elsewhere. Two Messages with identical content are equal. """ import base64 from dataclasses import dataclass, field from typing import Any from .models import Attachment from .serialization import ( AttachmentDict, AttachmentPartDict, MessageDict, PartDict, ReasoningPartDict, TextPartDict, ToolCallPartDict, ToolResultPartDict, ) def _attachment_to_dict(att: Attachment) -> AttachmentDict: d: dict[str, Any] = {} if att.type: d["type"] = att.type if att.url: d["url"] = att.url if att.path: d["path"] = att.path if att.content: d["content"] = base64.b64encode(att.content).decode("ascii") return d # type: ignore[return-value] def _attachment_from_dict(d: AttachmentDict) -> Attachment: raw_content = d.get("content") content_bytes: bytes | None = None if isinstance(raw_content, str): content_bytes = base64.b64decode(raw_content) return Attachment( type=d.get("type"), path=d.get("path"), url=d.get("url"), content=content_bytes, ) @dataclass class Part: """Base class for all parts. Role lives on the enclosing Message.""" def to_dict(self) -> PartDict: raise NotImplementedError @staticmethod def from_dict(d: PartDict) -> "Part": if d["type"] == "text": return TextPart( text=d["text"], provider_metadata=d.get("provider_metadata"), ) if d["type"] == "reasoning": return ReasoningPart( text=d["text"], redacted=d.get("redacted", False), provider_metadata=d.get("provider_metadata"), ) if d["type"] == "tool_call": return ToolCallPart( name=d["name"], arguments=d["arguments"], tool_call_id=d.get("tool_call_id"), server_executed=d.get("server_executed", False), provider_metadata=d.get("provider_metadata"), ) if d["type"] == "tool_result": return ToolResultPart( name=d["name"], output=d["output"], tool_call_id=d.get("tool_call_id"), server_executed=d.get("server_executed", False), exception=d.get("exception"), attachments=[ _attachment_from_dict(a) for a in d.get("attachments", []) ], provider_metadata=d.get("provider_metadata"), ) if d["type"] == "attachment": att_dict = d.get("attachment") attachment = _attachment_from_dict(att_dict) if att_dict else None return AttachmentPart( attachment=attachment, provider_metadata=d.get("provider_metadata"), ) raise ValueError(f"Unknown part type: {d['type']!r}") @dataclass class TextPart(Part): text: str = "" provider_metadata: dict[str, Any] | None = None def to_dict(self) -> TextPartDict: d: dict[str, Any] = {"type": "text", "text": self.text} if self.provider_metadata: d["provider_metadata"] = self.provider_metadata return d # type: ignore[return-value] @dataclass class ReasoningPart(Part): """Reasoning/thinking tokens from the model. `redacted=True, text=""` is the marker for the opaque-reasoning case (OpenAI GPT-5 series, Gemini without `includeThoughts`) where the provider reports that reasoning happened but withholds the content. The actual token total lives on response.token_details (e.g. `reasoning_tokens`); the Part only records the structural fact that reasoning occurred. """ text: str = "" redacted: bool = False provider_metadata: dict[str, Any] | None = None def to_dict(self) -> ReasoningPartDict: d: dict[str, Any] = {"type": "reasoning", "text": self.text} if self.redacted: d["redacted"] = True if self.provider_metadata: d["provider_metadata"] = self.provider_metadata return d # type: ignore[return-value] @dataclass class ToolCallPart(Part): """A request by the model to call a tool. `server_executed=True` marks calls the provider executed on the server (Anthropic web search, Gemini code execution) rather than the LLM tool framework. """ name: str = "" arguments: dict[str, Any] = field(default_factory=dict) tool_call_id: str | None = None server_executed: bool = False provider_metadata: dict[str, Any] | None = None def to_dict(self) -> ToolCallPartDict: d: dict[str, Any] = { "type": "tool_call", "name": self.name, "arguments": self.arguments, } if self.tool_call_id is not None: d["tool_call_id"] = self.tool_call_id if self.server_executed: d["server_executed"] = True if self.provider_metadata: d["provider_metadata"] = self.provider_metadata return d # type: ignore[return-value] @dataclass class ToolResultPart(Part): """The result of a tool call.""" name: str = "" output: str = "" tool_call_id: str | None = None server_executed: bool = False attachments: list[Any] = field(default_factory=list) exception: str | None = None provider_metadata: dict[str, Any] | None = None def to_dict(self) -> ToolResultPartDict: d: dict[str, Any] = { "type": "tool_result", "name": self.name, "output": self.output, } if self.tool_call_id is not None: d["tool_call_id"] = self.tool_call_id if self.server_executed: d["server_executed"] = True if self.exception is not None: d["exception"] = self.exception if self.attachments: d["attachments"] = [_attachment_to_dict(a) for a in self.attachments] if self.provider_metadata: d["provider_metadata"] = self.provider_metadata return d # type: ignore[return-value] @dataclass class AttachmentPart(Part): """An inline attachment (image, audio, file).""" attachment: Attachment | None = None provider_metadata: dict[str, Any] | None = None def to_dict(self) -> AttachmentPartDict: d: dict[str, Any] = {"type": "attachment"} if self.attachment: d["attachment"] = _attachment_to_dict(self.attachment) if self.provider_metadata: d["provider_metadata"] = self.provider_metadata return d # type: ignore[return-value] @dataclass class Message: """A single turn in a conversation: role + list of parts. `parts` contains one or more Part objects. `provider_metadata` carries opaque provider-specific data attached to the message as a whole; part-level data lives on the individual Part's `provider_metadata`. """ role: str parts: list[Part] = field(default_factory=list) provider_metadata: dict[str, Any] | None = None def to_dict(self) -> MessageDict: d: dict[str, Any] = { "role": self.role, "parts": [p.to_dict() for p in self.parts], } if self.provider_metadata: d["provider_metadata"] = self.provider_metadata return d # type: ignore[return-value] @staticmethod def from_dict(d: MessageDict) -> "Message": return Message( role=d["role"], parts=[Part.from_dict(p) for p in d.get("parts", [])], provider_metadata=d.get("provider_metadata"), ) def normalize_parts(items: Any) -> list[Part]: """Normalize helper inputs to a list of Part objects. Accepts str (→ TextPart), Attachment (→ AttachmentPart), Part (passed through), or a list/tuple of those (flattened one level). """ out: list[Part] = [] for item in items: if isinstance(item, Part): out.append(item) elif isinstance(item, str): out.append(TextPart(text=item)) elif isinstance(item, Attachment): out.append(AttachmentPart(attachment=item)) elif isinstance(item, (list, tuple)): out.extend(normalize_parts(item)) else: raise TypeError(f"Cannot convert {item!r} to an llm Part") return out def system(*items: Any, provider_metadata: dict[str, Any] | None = None) -> Message: "Build a Message with role='system'." return Message( role="system", parts=normalize_parts(items), provider_metadata=provider_metadata, ) def user(*items: Any, provider_metadata: dict[str, Any] | None = None) -> Message: "Build a Message with role='user'." return Message( role="user", parts=normalize_parts(items), provider_metadata=provider_metadata, ) def assistant(*items: Any, provider_metadata: dict[str, Any] | None = None) -> Message: "Build a Message with role='assistant'." return Message( role="assistant", parts=normalize_parts(items), provider_metadata=provider_metadata, ) def tool_message( *items: Any, provider_metadata: dict[str, Any] | None = None ) -> Message: "Build a Message with role='tool' (typically wrapping ToolResultParts)." return Message( role="tool", parts=normalize_parts(items), provider_metadata=provider_metadata, ) @dataclass class StreamEvent: """A streaming event from a model response. `part_index` groups events into parts. When left at its default of `None`, the framework allocates an index automatically: consecutive same-family text/reasoning events concatenate, tool-call events group by `tool_call_id`, and `tool_result` always starts its own part. Pass an explicit integer only to override the default grouping (e.g. forcing a single TextPart across non-adjacent text bursts). `redacted=True` (only meaningful on `type="reasoning"` events with an empty `chunk`) signals that opaque reasoning happened — content withheld by the provider, token total on response.token_details. The framework hoists redacted reasoning Parts to the start of the assembled message regardless of when they were emitted in the stream, so UIs can render them before the visible content. `provider_metadata` carries opaque provider data (Anthropic `signature`, Gemini `thoughtSignature`, OpenAI `encrypted_content`) that must be echoed back on the next request; the framework merges it onto the finalized Part (last non-None wins per top-level key). `message_index` is for providers that emit multiple assistant messages in a single response (OpenAI Responses server-side tool execution interleaves multiple `message` output items with tool calls); most plugins leave it at 0 and get a single assistant Message. Events with distinct indexes assemble into distinct Messages, in first-seen order. """ type: str # "text" / "reasoning" / "tool_call_name" / # "tool_call_args" / "tool_result" chunk: str part_index: int | None = None tool_call_id: str | None = None server_executed: bool = False tool_name: str | None = None redacted: bool = False provider_metadata: dict[str, Any] | None = None message_index: int = 0 ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/plugins.py0000644000175100017510000000312415234420023014502 0ustar00runnerrunnerimport importlib import os import sys from importlib import metadata import pluggy from . import hookspecs DEFAULT_PLUGINS = ( "llm.default_plugins.openai_models", "llm.default_plugins.default_tools", ) pm = pluggy.PluginManager("llm") pm.add_hookspecs(hookspecs) LLM_LOAD_PLUGINS = os.environ.get("LLM_LOAD_PLUGINS", None) _loaded = False def load_plugins(): global _loaded if _loaded: return _loaded = True if not hasattr(sys, "_called_from_test") and LLM_LOAD_PLUGINS is None: # Only load plugins if not running tests pm.load_setuptools_entrypoints("llm") # Load any plugins specified in LLM_LOAD_PLUGINS") if LLM_LOAD_PLUGINS is not None: for package_name in [ name for name in LLM_LOAD_PLUGINS.split(",") if name.strip() ]: try: distribution = metadata.distribution(package_name) # Updated call llm_entry_points = [ ep for ep in distribution.entry_points if ep.group == "llm" ] for entry_point in llm_entry_points: mod = entry_point.load() pm.register(mod, name=entry_point.name) # Ensure name can be found in plugin_to_distinfo later: pm._plugin_distinfo.append((mod, distribution)) # type: ignore except metadata.PackageNotFoundError: sys.stderr.write(f"Plugin {package_name} could not be found\n") for plugin in DEFAULT_PLUGINS: mod = importlib.import_module(plugin) pm.register(mod, plugin) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/py.typed0000644000175100017510000000000015234420023014134 0ustar00runnerrunner././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/serialization.py0000644000175100017510000001220015234420023015671 0ustar00runnerrunner"""TypedDict spec for the JSON-safe wire form of Part, Message, and Response. These are the exact shapes returned by ``Part.to_dict()``, ``Message.to_dict()``, and ``Response.to_dict()`` — and accepted by the matching ``from_dict`` classmethods. They are the canonical wire format; use them to annotate any code that reads or writes serialized llm data. Example:: from llm.serialization import MessageDict def save_messages(conn, messages: list[MessageDict]) -> None: for m in messages: conn.execute( "INSERT INTO messages(role, parts_json) VALUES (?, ?)", (m["role"], json.dumps(m["parts"])), ) Or pair with Pydantic's TypeAdapter for runtime validation:: from pydantic import TypeAdapter from llm.serialization import MessageDict msg = TypeAdapter(MessageDict).validate_python(incoming_dict) Or export JSON Schema for cross-language consumers:: schema = TypeAdapter(MessageDict).json_schema() The TypedDicts are erased at runtime — zero overhead. ``NotRequired`` keys may be absent from a serialized payload; required keys must always be present. """ from typing import Any, Literal # NotRequired moved to typing in 3.11; use typing_extensions for 3.10 # support. typing_extensions is a transitive dep via pydantic. from typing_extensions import NotRequired, TypedDict __all__ = [ "AttachmentDict", "AttachmentPartDict", "MessageDict", "PartDict", "PromptDict", "ReasoningPartDict", "ResponseDict", "TextPartDict", "ToolCallPartDict", "ToolResultPartDict", "UsageDict", ] # ---- Attachment payload (nested inside AttachmentPartDict + tool results) ---- class AttachmentDict(TypedDict, total=False): """Nested attachment payload. All fields optional — an Attachment may carry a type, a url, a path, and/or base64-encoded content. """ type: str url: str path: str # base64-encoded bytes when the attachment was constructed with raw # content= bytes. content: str # ---- Per-Part TypedDicts (discriminated by the `type` field) ----------------- class TextPartDict(TypedDict): type: Literal["text"] text: str provider_metadata: NotRequired[dict[str, Any]] class ReasoningPartDict(TypedDict): type: Literal["reasoning"] text: str # `redacted=True` with `text=""` is the marker for opaque # reasoning (OpenAI GPT-5, Gemini without thoughts). The token # total lives on response usage, not on the Part. redacted: NotRequired[bool] provider_metadata: NotRequired[dict[str, Any]] class ToolCallPartDict(TypedDict): type: Literal["tool_call"] name: str arguments: dict[str, Any] tool_call_id: NotRequired[str] # True for provider-executed calls (Anthropic web search, Gemini code # execution). Adapters use this to restore provider-side blocks on # the next turn. server_executed: NotRequired[bool] provider_metadata: NotRequired[dict[str, Any]] class ToolResultPartDict(TypedDict): type: Literal["tool_result"] name: str output: str tool_call_id: NotRequired[str] server_executed: NotRequired[bool] exception: NotRequired[str] attachments: NotRequired[list[AttachmentDict]] provider_metadata: NotRequired[dict[str, Any]] class AttachmentPartDict(TypedDict): type: Literal["attachment"] attachment: NotRequired[AttachmentDict] provider_metadata: NotRequired[dict[str, Any]] PartDict = ( TextPartDict | ReasoningPartDict | ToolCallPartDict | ToolResultPartDict | AttachmentPartDict ) """Discriminated union of Part dict shapes. Use with ``pydantic.TypeAdapter(PartDict)`` to validate / dispatch by ``type``. """ # ---- Message ---------------------------------------------------------------- class MessageDict(TypedDict): """JSON-safe form of ``llm.Message``. ``role`` is one of "user", "assistant", "system", "tool" in practice — typed as ``str`` here to leave room for provider-specific values. """ role: str parts: list[PartDict] provider_metadata: NotRequired[dict[str, Any]] # ---- Response + nested shapes ----------------------------------------------- class PromptDict(TypedDict): """The ``prompt`` sub-dict of ``Response.to_dict()`` — captures the full input chain that was sent for this turn plus any options that apply.""" messages: list[MessageDict] options: NotRequired[dict[str, Any]] system: NotRequired[str] class UsageDict(TypedDict, total=False): """Optional usage block on ``ResponseDict``. All fields optional; providers vary in which they report.""" input: int output: int details: dict[str, Any] class ResponseDict(TypedDict): """JSON-safe form of ``llm.Response`` — everything needed for ``Response.from_dict`` to rehydrate and ``response.reply()`` to continue a conversation across a process boundary. """ model: str prompt: PromptDict messages: list[MessageDict] # Audit fields — present on a freshly-serialized response, optional # on hand-constructed ones. id: NotRequired[str] usage: NotRequired[UsageDict] datetime_utc: NotRequired[str] ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/templates.py0000644000175100017510000000630715234420023015025 0ustar00runnerrunnerimport string from typing import Any from pydantic import BaseModel, ConfigDict class AttachmentType(BaseModel): type: str value: str class Template(BaseModel): """A reusable prompt template.""" name: str prompt: str | None = None system: str | None = None attachments: list[str] | None = None attachment_types: list[AttachmentType] | None = None model: str | None = None defaults: dict[str, Any] | None = None options: dict[str, Any] | None = None extract: bool | None = None # For extracting fenced code blocks extract_last: bool | None = None schema_object: dict | None = None fragments: list[str] | None = None system_fragments: list[str] | None = None tools: list[str] | None = None functions: str | None = None model_config = ConfigDict(extra="forbid") class MissingVariables(Exception): pass def __init__(self, **data): super().__init__(**data) # Not a pydantic field to avoid YAML being able to set it # this controls if Python inline functions code is trusted self._functions_is_trusted = False def evaluate( self, input: str, params: dict[str, Any] | None = None ) -> tuple[str | None, str | None]: """Evaluate the template with the given input and parameters, returning (prompt, system).""" params = params or {} params["input"] = input if self.defaults: for k, v in self.defaults.items(): if k not in params: params[k] = v prompt: str | None = None system: str | None = None if not self.prompt: system = self.interpolate(self.system, params) prompt = input else: prompt = self.interpolate(self.prompt, params) system = self.interpolate(self.system, params) return prompt, system def vars(self) -> set: """Return the set of variable names used in the prompt and system templates.""" all_vars = set() for text in [self.prompt, self.system]: if not text: continue all_vars.update(self.extract_vars(string.Template(text))) return all_vars @classmethod def interpolate(cls, text: str | None, params: dict[str, Any]) -> str | None: """Substitute template variables in text with values from params, raising MissingVariables if any are absent.""" if not text: return text # Confirm all variables in text are provided string_template = string.Template(text) vars = cls.extract_vars(string_template) missing = [p for p in vars if p not in params] if missing: raise cls.MissingVariables( "Missing variables: {}".format(", ".join(missing)) ) return string_template.substitute(**params) @staticmethod def extract_vars(string_template: string.Template) -> list[str]: """Extract and return the list of named variable identifiers from a string.Template.""" return [ match.group("named") for match in string_template.pattern.finditer(string_template.template) if match.group("named") ] ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/tools.py0000644000175100017510000000216015234420023014160 0ustar00runnerrunnerimport time from datetime import datetime, timezone from importlib.metadata import version def llm_version() -> str: "Return the installed version of llm" return version("llm") def llm_time() -> dict: "Returns the current time, as local time and UTC" # Get current times utc_time = datetime.now(timezone.utc) local_time = datetime.now(timezone.utc).astimezone() # Get timezone information local_tz_name = time.tzname[time.localtime().tm_isdst] is_dst = bool(time.localtime().tm_isdst) # Calculate offset offset_seconds = -time.timezone if not is_dst else -time.altzone offset_hours = offset_seconds // 3600 offset_minutes = (offset_seconds % 3600) // 60 timezone_offset = ( f"UTC{'+' if offset_hours >= 0 else ''}{offset_hours:02d}:{offset_minutes:02d}" ) return { "utc_time": utc_time.strftime("%Y-%m-%d %H:%M:%S UTC"), "utc_time_iso": utc_time.isoformat(), "local_timezone": local_tz_name, "local_time": local_time.strftime("%Y-%m-%d %H:%M:%S"), "timezone_offset": timezone_offset, "is_dst": is_dst, } ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/llm/utils.py0000644000175100017510000006064215234420023014171 0ustar00runnerrunnerimport hashlib import itertools import json import os import pathlib import re import textwrap import threading import time from typing import Any, Final import click import httpx import puremagic import sqlite_utils from ulid import ULID MIME_TYPE_FIXES = { "audio/wave": "audio/wav", } class Fragment(str): def __new__(cls, content, *args, **kwargs): # For immutable classes like str, __new__ creates the string object return super().__new__(cls, content) def __init__(self, content, source=""): # Initialize our custom attributes self.source = source def id(self): return hashlib.sha256(self.encode("utf-8")).hexdigest() def mimetype_from_string(content) -> str | None: try: type_ = puremagic.from_string(content, mime=True) return MIME_TYPE_FIXES.get(type_, type_) except puremagic.PureError: return None def mimetype_from_path(path) -> str | None: try: type_ = puremagic.from_file(path, mime=True) return MIME_TYPE_FIXES.get(type_, type_) except puremagic.PureError: return None def dicts_to_table_string( headings: list[str], dicts: list[dict[str, str]] ) -> list[str]: max_lengths = [len(h) for h in headings] # Compute maximum length for each column for d in dicts: for i, h in enumerate(headings): if h in d and len(str(d[h])) > max_lengths[i]: max_lengths[i] = len(str(d[h])) # Generate formatted table strings res = [] res.append(" ".join(h.ljust(max_lengths[i]) for i, h in enumerate(headings))) for d in dicts: row = [] for i, h in enumerate(headings): row.append(str(d.get(h, "")).ljust(max_lengths[i])) res.append(" ".join(row)) return res def remove_dict_none_values(d): """ Recursively remove keys with value of None or value of a dict that is all values of None """ if not isinstance(d, dict): return d new_dict = {} for key, value in d.items(): if value is not None: if isinstance(value, dict): nested = remove_dict_none_values(value) if nested: new_dict[key] = nested elif isinstance(value, list): new_dict[key] = [remove_dict_none_values(v) for v in value] else: new_dict[key] = value return new_dict class _LogResponse(httpx.Response): def iter_bytes(self, *args, **kwargs): for chunk in super().iter_bytes(*args, **kwargs): click.echo(chunk.decode(), err=True) yield chunk class _LogTransport(httpx.BaseTransport): def __init__(self, transport: httpx.BaseTransport): self.transport = transport def handle_request(self, request: httpx.Request) -> httpx.Response: response = self.transport.handle_request(request) return _LogResponse( status_code=response.status_code, headers=response.headers, stream=response.stream, extensions=response.extensions, ) def _no_accept_encoding(request: httpx.Request): request.headers.pop("accept-encoding", None) def _log_response(response: httpx.Response): request = response.request click.echo(f"Request: {request.method} {request.url}", err=True) click.echo(" Headers:", err=True) for key, value in request.headers.items(): if key.lower() == "authorization": value = "[...]" if key.lower() == "cookie": value = value.split("=")[0] + "=..." click.echo(f" {key}: {value}", err=True) click.echo(" Body:", err=True) try: request_body = json.loads(request.content) click.echo( textwrap.indent(json.dumps(request_body, indent=2), " "), err=True ) except json.JSONDecodeError: click.echo(textwrap.indent(request.content.decode(), " "), err=True) click.echo(f"Response: status_code={response.status_code}", err=True) click.echo(" Headers:", err=True) for key, value in response.headers.items(): if key.lower() == "set-cookie": value = value.split("=")[0] + "=..." click.echo(f" {key}: {value}", err=True) click.echo(" Body:", err=True) def logging_client() -> httpx.Client: return httpx.Client( transport=_LogTransport(httpx.HTTPTransport()), event_hooks={"request": [_no_accept_encoding], "response": [_log_response]}, ) def simplify_usage_dict(d): # Recursively remove keys with value 0 and empty dictionaries def remove_empty_and_zero(obj): if isinstance(obj, dict): cleaned = { k: remove_empty_and_zero(v) for k, v in obj.items() if v != 0 and v != {} } return {k: v for k, v in cleaned.items() if v is not None and v != {}} return obj return remove_empty_and_zero(d) or {} def token_usage_string(input_tokens, output_tokens, token_details) -> str: bits = [] if input_tokens is not None: bits.append(f"{format(input_tokens, ',')} input") if output_tokens is not None: bits.append(f"{format(output_tokens, ',')} output") if token_details: bits.append(json.dumps(token_details)) return ", ".join(bits) def extract_fenced_code_block(text: str, last: bool = False) -> str | None: """ Extracts and returns Markdown fenced code block found in the given text. The function handles fenced code blocks that: - Use at least three backticks (`). - May include a language tag immediately after the opening backticks. - Use more than three backticks as long as the closing fence has the same number. If no fenced code block is found, the function returns None. Args: text (str): The input text to search for a fenced code block. last (bool): Extract the last code block if True, otherwise the first. Returns: Optional[str]: The content of the fenced code block, or None if not found. """ # Regex pattern to match fenced code blocks # - ^ or \n ensures that the fence is at the start of a line # - (`{3,}) captures the opening backticks (at least three) # - (\w+)? optionally captures the language tag # - \n matches the newline after the opening fence # - (.*?) non-greedy match for the code block content # - (?P=fence) ensures that the closing fence has the same number of backticks # - [ ]* allows for optional spaces between the closing fence and newline # - (?=\n|$) ensures that the closing fence is followed by a newline or end of string pattern = re.compile( r"""(?m)^(?P`{3,})(?P\w+)?\n(?P.*?)^(?P=fence)[ ]*(?=\n|$)""", re.DOTALL, ) matches = list(pattern.finditer(text)) if matches: match = matches[-1] if last else matches[0] return match.group("code") return None def make_schema_id(schema: dict) -> tuple[str, str]: schema_json = json.dumps(schema, separators=(",", ":")) schema_id = hashlib.blake2b(schema_json.encode(), digest_size=16).hexdigest() return schema_id, schema_json def output_rows_as_json(rows, nl=False, compact=False, json_cols=()): """ Output rows as JSON - either newline-delimited or an array Parameters: - rows: Iterable of dictionaries to output - nl: Boolean, if True, use newline-delimited JSON - compact: Boolean, if True uses [{"...": "..."}\n {"...": "..."}] format - json_cols: Iterable of columns that contain JSON Yields: - Stream of strings to be output """ current_iter, next_iter = itertools.tee(rows, 2) next(next_iter, None) first = True for row, next_row in itertools.zip_longest(current_iter, next_iter): is_last = next_row is None for col in json_cols: row[col] = json.loads(row[col]) if nl: # Newline-delimited JSON: one JSON object per line yield json.dumps(row) elif compact: # Compact array format: [{"...": "..."}\n {"...": "..."}] yield "{firstchar}{serialized}{maybecomma}{lastchar}".format( firstchar="[" if first else " ", serialized=json.dumps(row), maybecomma="," if not is_last else "", lastchar="]" if is_last else "", ) else: # Pretty-printed array format with indentation yield "{firstchar}{serialized}{maybecomma}{lastchar}".format( firstchar="[\n" if first else "", serialized=textwrap.indent(json.dumps(row, indent=2), " "), maybecomma="," if not is_last else "", lastchar="\n]" if is_last else "", ) first = False if first and not nl: # We didn't output any rows, so yield the empty list yield "[]" def resolve_schema_input(db, schema_input, load_template): # schema_input might be JSON or a filepath or an ID or t:name if not schema_input: return if schema_input.strip().startswith("t:"): name = schema_input.strip()[2:] schema_object = None try: template = load_template(name) schema_object = template.schema_object except ValueError: raise click.ClickException(f"Invalid template: {name}") if not schema_object: raise click.ClickException(f"Template '{name}' has no schema") return template.schema_object if schema_input.strip().startswith("{"): try: return json.loads(schema_input) except ValueError: pass if " " in schema_input.strip() or "," in schema_input: # Treat it as schema DSL return schema_dsl(schema_input) # Is it a file on disk? path = pathlib.Path(schema_input) if path.exists(): try: return json.loads(path.read_text()) except ValueError: raise click.ClickException("Schema file contained invalid JSON") # Last attempt: is it an ID in the DB? try: row = db["schemas"].get(schema_input) return json.loads(row["content"]) except (sqlite_utils.db.NotFoundError, ValueError): raise click.BadParameter("Invalid schema") def schema_summary(schema: dict) -> str: """ Extract property names from a JSON schema and format them in a concise way that highlights the array/object structure. Args: schema (dict): A JSON schema dictionary Returns: str: A human-friendly summary of the schema structure """ if not schema or not isinstance(schema, dict): return "" schema_type = schema.get("type", "") if schema_type == "object": props = schema.get("properties", {}) prop_summaries = [] for name, prop_schema in props.items(): prop_type = prop_schema.get("type", "") if prop_type == "array": items = prop_schema.get("items", {}) items_summary = schema_summary(items) prop_summaries.append(f"{name}: [{items_summary}]") elif prop_type == "object": nested_summary = schema_summary(prop_schema) prop_summaries.append(f"{name}: {nested_summary}") else: prop_summaries.append(name) return "{" + ", ".join(prop_summaries) + "}" elif schema_type == "array": items = schema.get("items", {}) return schema_summary(items) return "" def schema_dsl(schema_dsl: str, multi: bool = False) -> dict[str, Any]: """ Build a JSON schema from a concise schema string. Args: schema_dsl: A string representing a schema in the concise format. Can be comma-separated or newline-separated. multi: Boolean, return a schema for an "items" array of these Returns: A dictionary representing the JSON schema. """ # Type mapping dictionary type_mapping = { "int": "integer", "float": "number", "bool": "boolean", "str": "string", } # Initialize the schema dictionary with required elements json_schema: dict[str, Any] = {"type": "object", "properties": {}, "required": []} # Check if the schema is newline-separated or comma-separated if "\n" in schema_dsl: fields = [field.strip() for field in schema_dsl.split("\n") if field.strip()] else: fields = [field.strip() for field in schema_dsl.split(",") if field.strip()] # Process each field for field in fields: # Extract field name, type, and description if ":" in field: field_info, description = field.split(":", 1) description = description.strip() else: field_info = field description = "" # Process field name and type field_parts = field_info.strip().split() if not field_parts: raise ValueError( f"Invalid schema DSL: field {field!r} is missing a name before ':'" ) field_name = field_parts[0].strip() # Default type is string field_type = "string" # If type is specified, use it if len(field_parts) > 1: type_indicator = field_parts[1].strip() if type_indicator in type_mapping: field_type = type_mapping[type_indicator] # Add field to properties json_schema["properties"][field_name] = {"type": field_type} # Add description if provided if description: json_schema["properties"][field_name]["description"] = description # Add field to required list json_schema["required"].append(field_name) if multi: return multi_schema(json_schema) else: return json_schema def multi_schema(schema: dict) -> dict: "Wrap JSON schema in an 'items': [] array" return { "type": "object", "properties": {"items": {"type": "array", "items": schema}}, "required": ["items"], } def find_unused_key(item: dict, key: str) -> str: 'Return unused key, e.g. for {"id": "1"} and key "id" returns "id_"' while key in item: key += "_" return key def truncate_string( text: str, max_length: int = 100, normalize_whitespace: bool = False, keep_end: bool = False, ) -> str: """ Truncate a string to a maximum length, with options to normalize whitespace and keep both start and end. Args: text: The string to truncate max_length: Maximum length of the result string normalize_whitespace: If True, replace all whitespace with a single space keep_end: If True, keep both beginning and end of string Returns: Truncated string """ if not text: return text if normalize_whitespace: text = re.sub(r"\s+", " ", text) if len(text) <= max_length: return text # Minimum sensible length for keep_end is 9 characters: "a... z" min_keep_end_length = 9 if keep_end and max_length >= min_keep_end_length: # Calculate how much text to keep at each end # Subtract 5 for the "... " separator cutoff = (max_length - 5) // 2 return text[:cutoff] + "... " + text[-cutoff:] else: # Fall back to simple truncation for very small max_length return text[: max_length - 3] + "..." def ensure_fragment(db, content): sql = """ insert into fragments (hash, content, datetime_utc, source) values (:hash, :content, datetime('now'), :source) on conflict(hash) do nothing """ hash_id = hashlib.sha256(content.encode("utf-8")).hexdigest() source = None if isinstance(content, Fragment): source = content.source db.execute(sql, {"hash": hash_id, "content": content, "source": source}) return db.execute( "select id from fragments where hash = :hash", {"hash": hash_id} ).fetchone()[0] def ensure_tool(db, tool): sql = """ insert into tools (hash, name, description, input_schema, plugin) values (:hash, :name, :description, :input_schema, :plugin) on conflict(hash) do nothing """ # No `with db.conn:` here - its exit commit would also commit any # open outer transaction, such as the one wrapping a turn write. db.execute( sql, { "hash": tool.hash(), "name": tool.name, "description": tool.description, "input_schema": json.dumps(tool.input_schema), "plugin": tool.plugin, }, ) return db.execute( "select id from tools where hash = :hash", {"hash": tool.hash()} ).fetchone()[0] def ensure_tool_instance(db, name, plugin, arguments) -> int: """Row id in tool_instances for this configuration, storing each distinct (plugin, name, arguments) once however many turns and calls it serves.""" match = db.execute( "select id from tool_instances where name is ? " "and plugin is ? and arguments is ?", [name, plugin, arguments], ).fetchone() if match: return match[0] return ( db["tool_instances"] .insert({"name": name, "plugin": plugin, "arguments": arguments}) .last_pk ) def maybe_fenced_code(content: str) -> str: "Return the content as a fenced code block if it looks like code" is_code = False if content.count("<") > 10: is_code = True if not is_code: # Are 90% of the lines under 120 chars? lines = content.splitlines() if len(lines) > 3: num_short = sum(1 for line in lines if len(line) < 120) if num_short / len(lines) > 0.9: is_code = True if is_code: # Find number of backticks not already present num_backticks = 3 while "`" * num_backticks in content: num_backticks += 1 # Add backticks content = ( "\n" + "`" * num_backticks + "\n" + content.strip() + "\n" + "`" * num_backticks ) return content _plugin_prefix_re = re.compile(r"^[a-zA-Z0-9_-]+:") def has_plugin_prefix(value: str) -> bool: "Check if value starts with alphanumeric prefix followed by a colon" return bool(_plugin_prefix_re.match(value)) def _parse_kwargs(arg_str: str) -> dict[str, Any]: """Parse key=value pairs where each value is valid JSON.""" tokens = [] buf = [] depth = 0 in_string = False string_char = "" escape = False for ch in arg_str: if in_string: buf.append(ch) if escape: escape = False elif ch == "\\": escape = True elif ch == string_char: in_string = False else: if ch in "\"'": in_string = True string_char = ch buf.append(ch) elif ch in "{[(": depth += 1 buf.append(ch) elif ch in "}])": depth -= 1 buf.append(ch) elif ch == "," and depth == 0: tokens.append("".join(buf).strip()) buf = [] else: buf.append(ch) if buf: tokens.append("".join(buf).strip()) kwargs: dict[str, Any] = {} for token in tokens: if not token: continue if "=" not in token: raise ValueError(f"Invalid keyword spec segment: '{token}'") key, value_str = token.split("=", 1) key = key.strip() value_str = value_str.strip() try: value = json.loads(value_str) except json.JSONDecodeError as e: raise ValueError(f"Value for '{key}' is not valid JSON: {value_str}") from e kwargs[key] = value return kwargs def instantiate_from_spec(class_map: dict[str, type], spec: str): """ Instantiate a class from a specification string with flexible argument formats. This function parses a specification string that defines a class name and its constructor arguments, then instantiates the class using the provided class mapping. The specification supports multiple argument formats for flexibility. Parameters ---------- class_map : Dict[str, Type] A mapping from class names (strings) to their corresponding class objects. Only classes present in this mapping can be instantiated. spec : str A specification string defining the class to instantiate and its arguments. Format: "ClassName" or "ClassName(arguments)" Supported argument formats: - Empty: ClassName() - calls constructor with no arguments - JSON object: ClassName({"key": "value", "other": 42}) - unpacked as **kwargs - Single JSON value: ClassName("hello") or ClassName([1,2,3]) - passed as single positional argument - Key-value pairs: ClassName(name="test", count=5, items=[1,2]) - parsed as individual kwargs where values must be valid JSON Returns ------- object An instance of the specified class, constructed with the parsed arguments. Raises ------ ValueError If the spec string format is invalid, if the class name is not found in class_map, if JSON parsing fails, or if argument parsing encounters errors. """ m = re.fullmatch(r"\s*([A-Za-z_][A-Za-z0-9_]*)\s*(?:\((.*)\))?\s*$", spec) if not m: raise ValueError(f"Invalid spec string: '{spec}'") class_name, arg_body = m.group(1), (m.group(2) or "").strip() if class_name not in class_map: raise ValueError(f"Unknown class '{class_name}'") cls = class_map[class_name] # No arguments at all if arg_body == "": return cls() # Starts with { -> JSON object to kwargs if arg_body.lstrip().startswith("{"): try: kw = json.loads(arg_body) except json.JSONDecodeError as e: raise ValueError("Argument JSON object is not valid JSON") from e if not isinstance(kw, dict): raise ValueError("Top-level JSON must be an object when using {} form") return cls(**kw) # Starts with quote / number / [ / t f n for single positional JSON value if re.match(r'\s*(["\[\d\-]|true|false|null)', arg_body, re.IGNORECASE): try: positional_value = json.loads(arg_body) except json.JSONDecodeError as e: raise ValueError("Positional argument must be valid JSON") from e return cls(positional_value) # Otherwise treat as key=value pairs kwargs = _parse_kwargs(arg_body) return cls(**kwargs) NANOSECS_IN_MILLISECS = 1000000 TIMESTAMP_LEN = 6 RANDOMNESS_LEN = 10 _lock: Final = threading.Lock() _last: bytes | None = None # 16-byte last produced ULID def monotonic_ulid() -> ULID: """ Return a ULID instance that is guaranteed to be *strictly larger* than every other ULID returned by this function inside the same process. It works the same way the reference JavaScript `monotonicFactory` does: * If the current call happens in the same millisecond as the previous one, the 80-bit randomness part is incremented by exactly one. * As soon as the system clock moves forward, a brand-new ULID with cryptographically secure randomness is generated. * If more than 2**80 ULIDs are requested within a single millisecond an `OverflowError` is raised (practically impossible). """ global _last now_ms = time.time_ns() // NANOSECS_IN_MILLISECS with _lock: # First call if _last is None: _last = _fresh(now_ms) return ULID(_last) # Decode timestamp from the last ULID we handed out last_ms = int.from_bytes(_last[:TIMESTAMP_LEN], "big") # If the millisecond is the same, increment the randomness if now_ms == last_ms: rand_int = int.from_bytes(_last[TIMESTAMP_LEN:], "big") + 1 if rand_int >= 1 << (RANDOMNESS_LEN * 8): raise OverflowError( "Randomness overflow: > 2**80 ULIDs requested " "in one millisecond!" ) randomness = rand_int.to_bytes(RANDOMNESS_LEN, "big") _last = _last[:TIMESTAMP_LEN] + randomness return ULID(_last) # New millisecond, start fresh _last = _fresh(now_ms) return ULID(_last) def _fresh(ms: int) -> bytes: """Build a brand-new 16-byte ULID for the given millisecond.""" timestamp = int.to_bytes(ms, TIMESTAMP_LEN, "big") randomness = os.urandom(RANDOMNESS_LEN) return timestamp + randomness ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1785864218.5843465 llm-0.32/llm.egg-info/0000755000175100017510000000000015234420033014142 5ustar00runnerrunner././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864218.0 llm-0.32/llm.egg-info/PKG-INFO0000644000175100017510000010255315234420032015244 0ustar00runnerrunnerMetadata-Version: 2.4 Name: llm Version: 0.32 Summary: CLI utility and Python library for interacting with Large Language Models from organizations like OpenAI, Anthropic and Gemini plus local models installed on your own machine. Author: Simon Willison License-Expression: Apache-2.0 Project-URL: Homepage, https://github.com/simonw/llm Project-URL: Documentation, https://llm.datasette.io/ Project-URL: Issues, https://github.com/simonw/llm/issues Project-URL: CI, https://github.com/simonw/llm/actions Project-URL: Changelog, https://github.com/simonw/llm/releases Classifier: Development Status :: 4 - Beta Classifier: Intended Audience :: Developers Classifier: Intended Audience :: End Users/Desktop Classifier: Intended Audience :: Science/Research Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: 3.13 Classifier: Programming Language :: Python :: 3.14 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence Classifier: Topic :: Text Processing :: Linguistic Classifier: Topic :: Utilities Requires-Python: >=3.10 Description-Content-Type: text/markdown License-File: LICENSE Requires-Dist: click Requires-Dist: condense-json>=1.1 Requires-Dist: openai>=2.32.0 Requires-Dist: click-default-group>=1.2.3 Requires-Dist: sqlite-utils>=4.0 Requires-Dist: pydantic>=2.0.0 Requires-Dist: PyYAML Requires-Dist: pluggy Requires-Dist: python-ulid Requires-Dist: setuptools Requires-Dist: pip Requires-Dist: pyreadline3; sys_platform == "win32" Requires-Dist: puremagic Dynamic: license-file # LLM [![GitHub repo](https://img.shields.io/badge/github-repo-green)](https://github.com/simonw/llm) [![PyPI](https://img.shields.io/pypi/v/llm.svg)](https://pypi.org/project/llm/) [![Changelog](https://img.shields.io/github/v/release/simonw/llm?include_prereleases&label=changelog)](https://llm.datasette.io/en/stable/changelog.html) [![Tests](https://github.com/simonw/llm/workflows/Test/badge.svg)](https://github.com/simonw/llm/actions?query=workflow%3ATest) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/llm/blob/main/LICENSE) [![Discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://datasette.io/discord-llm) [![Homebrew](https://img.shields.io/homebrew/installs/dy/llm?color=yellow&label=homebrew&logo=homebrew)](https://formulae.brew.sh/formula/llm) A CLI tool and Python library for interacting with **OpenAI**, **Anthropic’s Claude**, **Google’s Gemini**, **Meta’s Llama** and dozens of other Large Language Models, both via remote APIs and with models that can be installed and run on your own machine. Watch **[Language models on the command-line](https://www.youtube.com/watch?v=QUXQNi6jQ30)** on YouTube for a demo or [read the accompanying detailed notes](https://simonwillison.net/2024/Jun/17/cli-language-models/). With LLM you can: - [Run prompts from the command-line](https://llm.datasette.io/en/stable/usage.html#usage-executing-prompts) - [Store prompts and responses in SQLite](https://llm.datasette.io/en/stable/logging.html#logging) - [Generate and store embeddings](https://llm.datasette.io/en/stable/embeddings/index.html#embeddings) - [Extract structured content from text and images](https://llm.datasette.io/en/stable/schemas.html#schemas) - [Grant models the ability to execute tools](https://llm.datasette.io/en/stable/tools.html#tools) - … and much, much more ## Quick start First, install LLM using `pip` or Homebrew or `pipx` or `uv`: ```bash pip install llm ``` Or with Homebrew (see [warning note](https://llm.datasette.io/en/stable/setup.html#homebrew-warning)): ```bash brew install llm ``` Or with [pipx](https://pypa.github.io/pipx/): ```bash pipx install llm ``` Or with [uv](https://docs.astral.sh/uv/guides/tools/) ```bash uv tool install llm ``` Use LLM to run prompts or start chats against an arbitrary OpenAI-compatible Chat Completions endpoint, such as [LM Studio](https://lmstudio.ai). With `uvx`, you can do this without installing LLM first: ```bash uvx llm openai endpoint http://localhost:1234/v1 \ -m google/gemma-4-12b \ "What is the capital of France?" uvx llm openai endpoint http://localhost:1234/v1 \ -m google/gemma-4-12b \ --chat ``` Add `--key your-api-key` if the endpoint requires authentication. See [Run against an endpoint without configuring it](https://llm.datasette.io/en/stable/other-models.html#openai-endpoint) for more options. If you have an [OpenAI API key](https://platform.openai.com/api-keys) key you can run this: ```bash # Paste your OpenAI API key into this llm keys set openai # Run a prompt (with the default gpt-5.6-luna model) llm "Ten fun names for a pet pelican" # Extract text from an image llm "extract text" -a scanned-document.jpg # Use a system prompt against a file cat myfile.py | llm -s "Explain this code" ``` Run prompts against [Gemini](https://aistudio.google.com/apikey) or [Anthropic](https://console.anthropic.com/) with their respective plugins: ```bash llm install llm-gemini llm keys set gemini # Paste Gemini API key here llm -m gemini-3.5-flash 'Tell me fun facts about Mountain View' llm install llm-anthropic llm keys set anthropic # Paste Anthropic API key here llm -m claude-sonnet-5 'Impress me with wild facts about turnips' ``` You can also [install a plugin](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#installing-plugins) to access models that can run on your local device. If you use [Ollama](https://ollama.com/): ```bash # Install the plugin llm install llm-ollama # Download and run a prompt against the Orca Mini 7B model ollama pull llama3.2:latest llm -m llama3.2:latest 'What is the capital of France?' ``` To start [an interactive chat](https://llm.datasette.io/en/stable/usage.html#usage-chat) with a model, use `llm chat`: ```bash llm chat -m gpt-4.1 ``` ```default Chatting with gpt-4.1 Type 'exit' or 'quit' to exit Type '!multi' to enter multiple lines, then '!end' to finish Type '!edit' to open your default editor and modify the prompt. Type '!fragment [ ...]' to insert one or more fragments > Tell me a joke about a pelican Why don't pelicans like to tip waiters? Because they always have a big bill! ``` ## Project news - 29th April 2026: [LLM 0.32a0 is a major backwards-compatible refactor](https://simonwillison.net/2026/Apr/29/llm/) - 11th August 2025: [LLM 0.27, the annotated release notes: GPT-5 and improved tool calling](https://simonwillison.net/2025/Aug/11/llm-027/) - 27th May 2025: [Large Language Models can run tools in your terminal with LLM 0.26](https://simonwillison.net/2025/May/27/llm-tools/) - 5th May 2025: [Feed a video to a vision LLM as a sequence of JPEG frames on the CLI (also LLM 0.25)](https://simonwillison.net/2025/May/5/llm-video-frames/) - 7th April 2025: [Long context support in LLM 0.24 using fragments and template plugins](https://simonwillison.net/2025/Apr/7/long-context-llm/) - 28th February 2025: [Structured data extraction from unstructured content using LLM schemas](https://simonwillison.net/2025/Feb/28/llm-schemas/) - 17th February 2025: [LLM 0.22, the annotated release notes](https://simonwillison.net/2025/Feb/17/llm/) - 29th October 2024: [You can now run prompts against images, audio and video in your terminal using LLM](https://simonwillison.net/2024/Oct/29/llm-multi-modal/) - 26th January 2024: [LLM 0.13: The annotated release notes](https://simonwillison.net/2024/Jan/26/llm/) - 12th September 2023: [Build an image search engine with llm-clip, chat with models with llm chat](https://simonwillison.net/2023/Sep/12/llm-clip-and-chat/) - 4th September 2023: [LLM now provides tools for working with embeddings](https://simonwillison.net/2023/Sep/4/llm-embeddings/) - 12th July 2023: [The LLM CLI tool now supports self-hosted language models via plugins](https://simonwillison.net/2023/Jul/12/llm/) - 18th May 2023: [llm, ttok and strip-tags—CLI tools for working with ChatGPT and other LLMs](https://simonwillison.net/2023/May/18/cli-tools-for-llms/) - 4th April 2023: [The original announcement of the llm CLI tool](https://simonwillison.net/2023/Apr/4/llm/) For everything else, see [the llm tag](https://simonwillison.net/tags/llm/) on my blog. ## Contents * [Setup](https://llm.datasette.io/en/stable/setup.html) * [Installation](https://llm.datasette.io/en/stable/setup.html#installation) * [Upgrading to the latest version](https://llm.datasette.io/en/stable/setup.html#upgrading-to-the-latest-version) * [Using uvx](https://llm.datasette.io/en/stable/setup.html#using-uvx) * [A note about Homebrew and PyTorch](https://llm.datasette.io/en/stable/setup.html#a-note-about-homebrew-and-pytorch) * [Installing plugins](https://llm.datasette.io/en/stable/setup.html#installing-plugins) * [API key management](https://llm.datasette.io/en/stable/setup.html#api-key-management) * [Saving and using stored keys](https://llm.datasette.io/en/stable/setup.html#saving-and-using-stored-keys) * [Passing keys using the –key option](https://llm.datasette.io/en/stable/setup.html#passing-keys-using-the-key-option) * [Keys in environment variables](https://llm.datasette.io/en/stable/setup.html#keys-in-environment-variables) * [Configuration](https://llm.datasette.io/en/stable/setup.html#configuration) * [Setting a custom default model](https://llm.datasette.io/en/stable/setup.html#setting-a-custom-default-model) * [Setting a custom directory location](https://llm.datasette.io/en/stable/setup.html#setting-a-custom-directory-location) * [Turning SQLite logging on and off](https://llm.datasette.io/en/stable/setup.html#turning-sqlite-logging-on-and-off) * [Usage](https://llm.datasette.io/en/stable/usage.html) * [Executing a prompt](https://llm.datasette.io/en/stable/usage.html#executing-a-prompt) * [Model options](https://llm.datasette.io/en/stable/usage.html#model-options) * [Attachments](https://llm.datasette.io/en/stable/usage.html#attachments) * [System prompts](https://llm.datasette.io/en/stable/usage.html#system-prompts) * [Tools](https://llm.datasette.io/en/stable/usage.html#tools) * [Extracting fenced code blocks](https://llm.datasette.io/en/stable/usage.html#extracting-fenced-code-blocks) * [JSON output](https://llm.datasette.io/en/stable/usage.html#json-output) * [Schemas](https://llm.datasette.io/en/stable/usage.html#schemas) * [Fragments](https://llm.datasette.io/en/stable/usage.html#fragments) * [Continuing a conversation](https://llm.datasette.io/en/stable/usage.html#continuing-a-conversation) * [Tips for using LLM with Bash or Zsh](https://llm.datasette.io/en/stable/usage.html#tips-for-using-llm-with-bash-or-zsh) * [Completion prompts](https://llm.datasette.io/en/stable/usage.html#completion-prompts) * [Starting an interactive chat](https://llm.datasette.io/en/stable/usage.html#starting-an-interactive-chat) * [Listing available models](https://llm.datasette.io/en/stable/usage.html#listing-available-models) * [Setting default options for models](https://llm.datasette.io/en/stable/usage.html#setting-default-options-for-models) * [OpenAI models](https://llm.datasette.io/en/stable/openai-models.html) * [Configuration](https://llm.datasette.io/en/stable/openai-models.html#configuration) * [OpenAI language models](https://llm.datasette.io/en/stable/openai-models.html#openai-language-models) * [Model features](https://llm.datasette.io/en/stable/openai-models.html#model-features) * [Web Search](https://llm.datasette.io/en/stable/openai-models.html#web-search) * [Code Interpreter](https://llm.datasette.io/en/stable/openai-models.html#code-interpreter) * [Fast mode and service tiers](https://llm.datasette.io/en/stable/openai-models.html#fast-mode-and-service-tiers) * [OpenAI embedding models](https://llm.datasette.io/en/stable/openai-models.html#openai-embedding-models) * [OpenAI completion models](https://llm.datasette.io/en/stable/openai-models.html#openai-completion-models) * [Adding more OpenAI models](https://llm.datasette.io/en/stable/openai-models.html#adding-more-openai-models) * [Other models](https://llm.datasette.io/en/stable/other-models.html) * [Installing and using a local model](https://llm.datasette.io/en/stable/other-models.html#installing-and-using-a-local-model) * [OpenAI-compatible models](https://llm.datasette.io/en/stable/other-models.html#openai-compatible-models) * [Run against an endpoint without configuring it](https://llm.datasette.io/en/stable/other-models.html#run-against-an-endpoint-without-configuring-it) * [Configure an OpenAI-compatible model](https://llm.datasette.io/en/stable/other-models.html#configure-an-openai-compatible-model) * [Extra HTTP headers](https://llm.datasette.io/en/stable/other-models.html#extra-http-headers) * [Tools](https://llm.datasette.io/en/stable/tools.html) * [How tools work](https://llm.datasette.io/en/stable/tools.html#how-tools-work) * [Trying out tools](https://llm.datasette.io/en/stable/tools.html#trying-out-tools) * [LLM’s implementation of tools](https://llm.datasette.io/en/stable/tools.html#llm-s-implementation-of-tools) * [Default tools](https://llm.datasette.io/en/stable/tools.html#default-tools) * [Tips for implementing tools](https://llm.datasette.io/en/stable/tools.html#tips-for-implementing-tools) * [Schemas](https://llm.datasette.io/en/stable/schemas.html) * [Schemas tutorial](https://llm.datasette.io/en/stable/schemas.html#schemas-tutorial) * [Getting started with dogs](https://llm.datasette.io/en/stable/schemas.html#getting-started-with-dogs) * [Extracting people from a news articles](https://llm.datasette.io/en/stable/schemas.html#extracting-people-from-a-news-articles) * [Using JSON schemas](https://llm.datasette.io/en/stable/schemas.html#using-json-schemas) * [Ways to specify a schema](https://llm.datasette.io/en/stable/schemas.html#ways-to-specify-a-schema) * [Concise LLM schema syntax](https://llm.datasette.io/en/stable/schemas.html#concise-llm-schema-syntax) * [Saving reusable schemas in templates](https://llm.datasette.io/en/stable/schemas.html#saving-reusable-schemas-in-templates) * [Browsing logged JSON objects created using schemas](https://llm.datasette.io/en/stable/schemas.html#browsing-logged-json-objects-created-using-schemas) * [Templates](https://llm.datasette.io/en/stable/templates.html) * [Getting started with –save](https://llm.datasette.io/en/stable/templates.html#getting-started-with-save) * [Using a template](https://llm.datasette.io/en/stable/templates.html#using-a-template) * [Listing available templates](https://llm.datasette.io/en/stable/templates.html#listing-available-templates) * [Templates as YAML files](https://llm.datasette.io/en/stable/templates.html#templates-as-yaml-files) * [System prompts](https://llm.datasette.io/en/stable/templates.html#system-prompts) * [Fragments](https://llm.datasette.io/en/stable/templates.html#fragments) * [Options](https://llm.datasette.io/en/stable/templates.html#options) * [Tools](https://llm.datasette.io/en/stable/templates.html#tools) * [Schemas](https://llm.datasette.io/en/stable/templates.html#schemas) * [Additional template variables](https://llm.datasette.io/en/stable/templates.html#additional-template-variables) * [Specifying default parameters](https://llm.datasette.io/en/stable/templates.html#specifying-default-parameters) * [Configuring code extraction](https://llm.datasette.io/en/stable/templates.html#configuring-code-extraction) * [Setting a default model for a template](https://llm.datasette.io/en/stable/templates.html#setting-a-default-model-for-a-template) * [Template loaders from plugins](https://llm.datasette.io/en/stable/templates.html#template-loaders-from-plugins) * [Fragments](https://llm.datasette.io/en/stable/fragments.html) * [Using fragments in a prompt](https://llm.datasette.io/en/stable/fragments.html#using-fragments-in-a-prompt) * [Using fragments in chat](https://llm.datasette.io/en/stable/fragments.html#using-fragments-in-chat) * [Browsing fragments](https://llm.datasette.io/en/stable/fragments.html#browsing-fragments) * [Setting aliases for fragments](https://llm.datasette.io/en/stable/fragments.html#setting-aliases-for-fragments) * [Viewing fragments in your logs](https://llm.datasette.io/en/stable/fragments.html#viewing-fragments-in-your-logs) * [Using fragments from plugins](https://llm.datasette.io/en/stable/fragments.html#using-fragments-from-plugins) * [Listing available fragment prefixes](https://llm.datasette.io/en/stable/fragments.html#listing-available-fragment-prefixes) * [Model aliases](https://llm.datasette.io/en/stable/aliases.html) * [Listing aliases](https://llm.datasette.io/en/stable/aliases.html#listing-aliases) * [Adding a new alias](https://llm.datasette.io/en/stable/aliases.html#adding-a-new-alias) * [Removing an alias](https://llm.datasette.io/en/stable/aliases.html#removing-an-alias) * [Viewing the aliases file](https://llm.datasette.io/en/stable/aliases.html#viewing-the-aliases-file) * [Embeddings](https://llm.datasette.io/en/stable/embeddings/index.html) * [Embedding with the CLI](https://llm.datasette.io/en/stable/embeddings/cli.html) * [llm embed](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed) * [llm embed-multi](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed-multi) * [llm similar](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-similar) * [llm embed-models](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-embed-models) * [llm collections list](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-collections-list) * [llm collections delete](https://llm.datasette.io/en/stable/embeddings/cli.html#llm-collections-delete) * [Using embeddings from Python](https://llm.datasette.io/en/stable/embeddings/python-api.html) * [Working with collections](https://llm.datasette.io/en/stable/embeddings/python-api.html#working-with-collections) * [Retrieving similar items](https://llm.datasette.io/en/stable/embeddings/python-api.html#retrieving-similar-items) * [SQL schema](https://llm.datasette.io/en/stable/embeddings/python-api.html#sql-schema) * [Writing plugins to add new embedding models](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html) * [`EmbeddingModel`](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html#llm.EmbeddingModel) * [Embedding binary content](https://llm.datasette.io/en/stable/embeddings/writing-plugins.html#embedding-binary-content) * [Embedding storage format](https://llm.datasette.io/en/stable/embeddings/storage.html) * [Plugins](https://llm.datasette.io/en/stable/plugins/index.html) * [Installing plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html) * [Listing installed plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#listing-installed-plugins) * [Running with a subset of plugins](https://llm.datasette.io/en/stable/plugins/installing-plugins.html#running-with-a-subset-of-plugins) * [Plugin directory](https://llm.datasette.io/en/stable/plugins/directory.html) * [Local models](https://llm.datasette.io/en/stable/plugins/directory.html#local-models) * [Remote APIs](https://llm.datasette.io/en/stable/plugins/directory.html#remote-apis) * [Tools](https://llm.datasette.io/en/stable/plugins/directory.html#tools) * [Fragments and template loaders](https://llm.datasette.io/en/stable/plugins/directory.html#fragments-and-template-loaders) * [Embedding models](https://llm.datasette.io/en/stable/plugins/directory.html#embedding-models) * [Extra commands](https://llm.datasette.io/en/stable/plugins/directory.html#extra-commands) * [Just for fun](https://llm.datasette.io/en/stable/plugins/directory.html#just-for-fun) * [Plugin hooks](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html) * [register_commands(cli)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-commands-cli) * [register_models(register, model_aliases)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-models-register-model-aliases) * [register_embedding_models(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-embedding-models-register) * [register_tools(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-tools-register) * [register_template_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-template-loaders-register) * [register_fragment_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-fragment-loaders-register) * [Developing a model plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html) * [The initial structure of the plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#the-initial-structure-of-the-plugin) * [Installing your plugin to try it out](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#installing-your-plugin-to-try-it-out) * [Building the Markov chain](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#building-the-markov-chain) * [Executing the Markov chain](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#executing-the-markov-chain) * [Adding that to the plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-that-to-the-plugin) * [Understanding execute()](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#understanding-execute) * [Prompts and responses are logged to the database](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#prompts-and-responses-are-logged-to-the-database) * [Adding options](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-options) * [Distributing your plugin](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#distributing-your-plugin) * [GitHub repositories](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#github-repositories) * [Publishing plugins to PyPI](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#publishing-plugins-to-pypi) * [Adding metadata](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#adding-metadata) * [What to do if it breaks](https://llm.datasette.io/en/stable/plugins/tutorial-model-plugin.html#what-to-do-if-it-breaks) * [Advanced model plugins](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html) * [Tip: lazily load expensive dependencies](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tip-lazily-load-expensive-dependencies) * [Models that accept API keys](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#models-that-accept-api-keys) * [Async models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#async-models) * [Supporting schemas](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-schemas) * [Supporting tools](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-tools) * [Supporting server-side tools](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-server-side-tools) * [Attachments for multi-modal models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#attachments-for-multi-modal-models) * [Structured messages and streaming events](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#structured-messages-and-streaming-events) * [Condensing logged payloads with json_replacements](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#condensing-logged-payloads-with-json-replacements) * [Consuming prompt.messages in build_messages](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#consuming-prompt-messages-in-build-messages) * [Restoring opaque metadata on subsequent requests](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#restoring-opaque-metadata-on-subsequent-requests) * [Tracking token usage](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-token-usage) * [Tracking resolved model names](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-resolved-model-names) * [LLM_RAISE_ERRORS](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#llm-raise-errors) * [Utility functions for plugins](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html) * [llm.get_key()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-get-key) * [llm.user_dir()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-user-dir) * [llm.ModelError](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-modelerror) * [Response.fake()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#response-fake) * [Python API](https://llm.datasette.io/en/stable/python-api.html) * [Basic prompt execution](https://llm.datasette.io/en/stable/python-api.html#basic-prompt-execution) * [System prompts](https://llm.datasette.io/en/stable/python-api.html#system-prompts) * [Attachments](https://llm.datasette.io/en/stable/python-api.html#attachments) * [Tools](https://llm.datasette.io/en/stable/python-api.html#tools) * [Schemas](https://llm.datasette.io/en/stable/python-api.html#schemas) * [Fragments](https://llm.datasette.io/en/stable/python-api.html#fragments) * [Model options](https://llm.datasette.io/en/stable/python-api.html#model-options) * [Passing an API key](https://llm.datasette.io/en/stable/python-api.html#passing-an-api-key) * [Models from plugins](https://llm.datasette.io/en/stable/python-api.html#models-from-plugins) * [Accessing the underlying JSON](https://llm.datasette.io/en/stable/python-api.html#accessing-the-underlying-json) * [Token usage](https://llm.datasette.io/en/stable/python-api.html#token-usage) * [Streaming responses](https://llm.datasette.io/en/stable/python-api.html#streaming-responses) * [Structured messages and streaming events](https://llm.datasette.io/en/stable/python-api.html#structured-messages-and-streaming-events) * [Async models](https://llm.datasette.io/en/stable/python-api.html#async-models) * [`AsyncResponse`](https://llm.datasette.io/en/stable/python-api.html#llm.AsyncResponse) * [Tool functions can be sync or async](https://llm.datasette.io/en/stable/python-api.html#tool-functions-can-be-sync-or-async) * [Tool use for async models](https://llm.datasette.io/en/stable/python-api.html#tool-use-for-async-models) * [Conversations](https://llm.datasette.io/en/stable/python-api.html#conversations) * [Conversations using tools](https://llm.datasette.io/en/stable/python-api.html#conversations-using-tools) * [Listing models](https://llm.datasette.io/en/stable/python-api.html#listing-models) * [Running code when a response has completed](https://llm.datasette.io/en/stable/python-api.html#running-code-when-a-response-has-completed) * [Other functions](https://llm.datasette.io/en/stable/python-api.html#other-functions) * [set_alias(alias, model_id)](https://llm.datasette.io/en/stable/python-api.html#set-alias-alias-model-id) * [remove_alias(alias)](https://llm.datasette.io/en/stable/python-api.html#remove-alias-alias) * [set_default_model(alias)](https://llm.datasette.io/en/stable/python-api.html#set-default-model-alias) * [get_default_model()](https://llm.datasette.io/en/stable/python-api.html#get-default-model) * [set_default_embedding_model(alias) and get_default_embedding_model()](https://llm.datasette.io/en/stable/python-api.html#set-default-embedding-model-alias-and-get-default-embedding-model) * [Logging to SQLite](https://llm.datasette.io/en/stable/logging.html) * [Viewing the logs](https://llm.datasette.io/en/stable/logging.html#viewing-the-logs) * [-s/–short mode](https://llm.datasette.io/en/stable/logging.html#s-short-mode) * [Logs for a conversation](https://llm.datasette.io/en/stable/logging.html#logs-for-a-conversation) * [Searching the logs](https://llm.datasette.io/en/stable/logging.html#searching-the-logs) * [Filtering past a specific ID](https://llm.datasette.io/en/stable/logging.html#filtering-past-a-specific-id) * [Filtering by model](https://llm.datasette.io/en/stable/logging.html#filtering-by-model) * [Filtering by prompts that used specific fragments](https://llm.datasette.io/en/stable/logging.html#filtering-by-prompts-that-used-specific-fragments) * [Filtering by prompts that used specific tools](https://llm.datasette.io/en/stable/logging.html#filtering-by-prompts-that-used-specific-tools) * [Browsing data collected using schemas](https://llm.datasette.io/en/stable/logging.html#browsing-data-collected-using-schemas) * [Browsing logs using Datasette](https://llm.datasette.io/en/stable/logging.html#browsing-logs-using-datasette) * [Backing up your database](https://llm.datasette.io/en/stable/logging.html#backing-up-your-database) * [The message store](https://llm.datasette.io/en/stable/logging.html#the-message-store) * [Threads, turns, messages and parts](https://llm.datasette.io/en/stable/logging.html#threads-turns-messages-and-parts) * [A worked example](https://llm.datasette.io/en/stable/logging.html#a-worked-example) * [Content addressing as a contract](https://llm.datasette.io/en/stable/logging.html#content-addressing-as-a-contract) * [Forking and shared history](https://llm.datasette.io/en/stable/logging.html#forking-and-shared-history) * [Storage by reference](https://llm.datasette.io/en/stable/logging.html#storage-by-reference) * [The raw provider payload](https://llm.datasette.io/en/stable/logging.html#the-raw-provider-payload) * [Table by table](https://llm.datasette.io/en/stable/logging.html#table-by-table) * [Querying the message store](https://llm.datasette.io/en/stable/logging.html#querying-the-message-store) * [Logging from Python](https://llm.datasette.io/en/stable/logging.html#logging-from-python) * [SQL schema](https://llm.datasette.io/en/stable/logging.html#sql-schema) * [Related tools](https://llm.datasette.io/en/stable/related-tools.html) * [strip-tags](https://llm.datasette.io/en/stable/related-tools.html#strip-tags) * [ttok](https://llm.datasette.io/en/stable/related-tools.html#ttok) * [Symbex](https://llm.datasette.io/en/stable/related-tools.html#symbex) * [CLI reference](https://llm.datasette.io/en/stable/help.html) * [llm –help](https://llm.datasette.io/en/stable/help.html#llm-help) * [llm prompt –help](https://llm.datasette.io/en/stable/help.html#llm-prompt-help) * [llm chat –help](https://llm.datasette.io/en/stable/help.html#llm-chat-help) * [llm keys –help](https://llm.datasette.io/en/stable/help.html#llm-keys-help) * [llm logs –help](https://llm.datasette.io/en/stable/help.html#llm-logs-help) * [llm models –help](https://llm.datasette.io/en/stable/help.html#llm-models-help) * [llm templates –help](https://llm.datasette.io/en/stable/help.html#llm-templates-help) * [llm schemas –help](https://llm.datasette.io/en/stable/help.html#llm-schemas-help) * [llm tools –help](https://llm.datasette.io/en/stable/help.html#llm-tools-help) * [llm aliases –help](https://llm.datasette.io/en/stable/help.html#llm-aliases-help) * [llm fragments –help](https://llm.datasette.io/en/stable/help.html#llm-fragments-help) * [llm plugins –help](https://llm.datasette.io/en/stable/help.html#llm-plugins-help) * [llm install –help](https://llm.datasette.io/en/stable/help.html#llm-install-help) * [llm uninstall –help](https://llm.datasette.io/en/stable/help.html#llm-uninstall-help) * [llm embed –help](https://llm.datasette.io/en/stable/help.html#llm-embed-help) * [llm embed-multi –help](https://llm.datasette.io/en/stable/help.html#llm-embed-multi-help) * [llm similar –help](https://llm.datasette.io/en/stable/help.html#llm-similar-help) * [llm embed-models –help](https://llm.datasette.io/en/stable/help.html#llm-embed-models-help) * [llm collections –help](https://llm.datasette.io/en/stable/help.html#llm-collections-help) * [llm openai –help](https://llm.datasette.io/en/stable/help.html#llm-openai-help) * [Contributing](https://llm.datasette.io/en/stable/contributing.html) * [Updating recorded HTTP API interactions and associated snapshots](https://llm.datasette.io/en/stable/contributing.html#updating-recorded-http-api-interactions-and-associated-snapshots) * [Debugging tricks](https://llm.datasette.io/en/stable/contributing.html#debugging-tricks) * [Documentation](https://llm.datasette.io/en/stable/contributing.html#documentation) * [Release process](https://llm.datasette.io/en/stable/contributing.html#release-process) * [Changelog](https://llm.datasette.io/en/stable/changelog.html) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864218.0 llm-0.32/llm.egg-info/SOURCES.txt0000644000175100017510000000111015234420032016016 0ustar00runnerrunnerLICENSE MANIFEST.in README.md pyproject.toml llm/__init__.py llm/__main__.py llm/cli.py llm/embeddings.py llm/embeddings_migrations.py llm/errors.py llm/hookspecs.py llm/logs.py llm/migrations.py llm/models.py llm/parts.py llm/plugins.py llm/py.typed llm/serialization.py llm/templates.py llm/tools.py llm/utils.py llm.egg-info/PKG-INFO llm.egg-info/SOURCES.txt llm.egg-info/dependency_links.txt llm.egg-info/entry_points.txt llm.egg-info/requires.txt llm.egg-info/top_level.txt llm/default_plugins/__init__.py llm/default_plugins/default_tools.py llm/default_plugins/openai_models.py././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864218.0 llm-0.32/llm.egg-info/dependency_links.txt0000644000175100017510000000000115234420032020207 0ustar00runnerrunner ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864218.0 llm-0.32/llm.egg-info/entry_points.txt0000644000175100017510000000004415234420032017435 0ustar00runnerrunner[console_scripts] llm = llm.cli:cli ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864218.0 llm-0.32/llm.egg-info/requires.txt0000644000175100017510000000030015234420032016532 0ustar00runnerrunnerclick condense-json>=1.1 openai>=2.32.0 click-default-group>=1.2.3 sqlite-utils>=4.0 pydantic>=2.0.0 PyYAML pluggy python-ulid setuptools pip puremagic [:sys_platform == "win32"] pyreadline3 ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864218.0 llm-0.32/llm.egg-info/top_level.txt0000644000175100017510000000000415234420032016665 0ustar00runnerrunnerllm ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1785864211.0 llm-0.32/pyproject.toml0000644000175100017510000000424515234420023014604 0ustar00runnerrunner[project] name = "llm" version = "0.32" description = "CLI utility and Python library for interacting with Large Language Models from organizations like OpenAI, Anthropic and Gemini plus local models installed on your own machine." readme = { file = "README.md", content-type = "text/markdown" } authors = [ { name = "Simon Willison" }, ] license = "Apache-2.0" requires-python = ">=3.10" classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Intended Audience :: End Users/Desktop", "Intended Audience :: Science/Research", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Text Processing :: Linguistic", "Topic :: Utilities", ] dependencies = [ "click", "condense-json>=1.1", "openai>=2.32.0", "click-default-group>=1.2.3", "sqlite-utils>=4.0", "pydantic>=2.0.0", "PyYAML", "pluggy", "python-ulid", "setuptools", "pip", "pyreadline3; sys_platform == 'win32'", "puremagic", ] [dependency-groups] dev = [ "build", "click>=8.2.0", "pytest", "numpy", "pytest-httpx>=0.33.0", "pytest-asyncio", "cogapp", "mypy>=1.10.0", "black>=26.3.1", "pytest-recording", "ruff>=0.16.0", "syrupy", "types-click", "types-PyYAML", "types-setuptools", "llm-echo==0.3a3", # docs "sphinx==7.2.6", "furo==2023.9.10", "sphinx-autobuild", "sphinx-copybutton", "sphinx-markdown-builder==0.6.8", "myst-parser", ] [project.urls] Homepage = "https://github.com/simonw/llm" Documentation = "https://llm.datasette.io/" Issues = "https://github.com/simonw/llm/issues" CI = "https://github.com/simonw/llm/actions" Changelog = "https://github.com/simonw/llm/releases" [project.scripts] llm = "llm.cli:cli" [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] include = ["llm*"] ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1785864218.5853977 llm-0.32/setup.cfg0000644000175100017510000000004615234420033013505 0ustar00runnerrunner[egg_info] tag_build = tag_date = 0